版权声明：本文为博主原创文章，转载请在显著位置标明本文出处以及作者网名，未经作者允许不得用于商业目的。
EmguCV是一个基于OpenCV的开源免费的跨平台计算机视觉库,它向C#和VB.NET开发者提供了OpenCV库的大部分功能。
教程VB.net版本请访问：EmguCV学习笔记 VB.Net 目录-CSDN博客
教程C#版本请访问：EmguCV学习笔记 C# 目录-CSDN博客
笔者的博客网址：https://blog.csdn.net/uruseibest
教程配套文件及相关说明以及如何获得pdf教程和代码，请移步：EmguCV学习笔记
学习VB.Net知识，请移步： vb.net 教程 目录_vb中如何用datagridview-CSDN博客
学习C#知识，请移步：C# 教程 目录_c#教程目录-CSDN博客
11.6 图像分割
11.6.1 语义分割Fcn
FCN（Fully Convolutional Network）是一种流行的语义分割算法，它通过将传统的卷积神经网络转化为全卷积网络来实现像素级别的语义分割，将输入图像传入网络中，得到每个像素的标签，并根据标签来分割出不同的物体。FCN在语义分割任务中表现优秀，可以实现像素级别的精细分割，被广泛应用于自动驾驶、智能监控等领域。
【代码位置：frmChapter11】Button5_Click、getColorTable、showColorTable
'语义分割fcn
Private Sub Button5_Click(sender As Object, e As EventArgs) Handles Button5.Click
'对象分类，object_detection_classes_pascal_voc.txt文件提供了21类对象（含background）
Dim classnames() As String
classnames = File.ReadAllLines("C:\learnEmgucv\fcn\object_detection_classes_pascal_voc.txt")
'获得输出颜色表
Dim colorTable() As Bgr = getColorTable()
'显示颜色表及对应对象名称
Call showColorTable(classnames, colorTable)
'需要测试的图像文件
Dim m As New Mat("C:\learnEmgucv\dnntest1.jpg", ImreadModes.Color)
Dim hm As Single = m.Height
Dim wm As Single = m.Width
Dim net As Dnn.Net
net = DnnInvoke.ReadNetFromCaffe("C:\learnEmgucv\fcn\fcn8s-heavy-pascal.prototxt",
"C:\learnEmgucv\fcn\fcn8s-heavy-pascal.caffemodel"
)
Dim mcopy As New Mat
CvInvoke.Resize(m, mcopy, New Drawing.Size(500, 500))
Dim blob As Mat
blob = DnnInvoke.BlobFromImage(mcopy, 1, New Size(500, 500),
New MCvScalar(0， 0， 0), False, False)
net.SetInput(blob)
'
Dim mout As New Mat
mout = net.Forward()
Dim fout(,,,) As Single
fout = mout.GetData()
'通道数=21，即21类对象
Dim chan As Integer = fout.GetLength(1)
'高度=500
Dim row As Integer = fout.GetLength(2)
'宽度=500
Dim col As Integer = fout.GetLength(3)
'记录21个通道对应坐标点的最大值
Dim matrMaxValue As New Matrix(Of Single)(New Size(row, col))
matrMaxValue.SetZero()
''记录对应通道号
'Dim matrMaxChan As New Matrix(Of Single)(New Size(row, col))
'matrMaxChan.SetZero()
'记录对应颜色
Dim imgOut As New Image(Of Bgr, Byte)(col, row)
imgOut.SetZero()
'遍历21个通道
For c As Integer = 0 To chan - 1
For h As Integer = 0 To row - 1
'遍历高度和宽度，获得对应坐标的值
For w As Integer = 0 To col - 1
'比较最大值
If fout(0, c, h, w) > matrMaxValue(h, w) Then
'取得最大值
matrMaxValue(h, w) = fout(0, c, h, w)
''取得通道号
'matrMaxChan(h, w) = c
'最重要的是获取通道对应颜色表的值
imgOut(h, w) = colorTable(c)
End If
Next
Next
Next
'显示输出的图像
'ImageBox1.Image = imgOut.Mat
'设置掩膜
Dim mask As New Mat
mask = imgOut.Mat
'掩膜大小必须和源图像一致
CvInvoke.Resize(mask, mask, m.Size)
'将上面的输出图像和源图像叠加
Dim mFinalOut As New Mat
CvInvoke.AddWeighted(m, 0.3, mask, 0.7, 0, mFinalOut)
ImageBox1.Image = mFinalOut
End Sub
'为了更好地观察，这里没有使用随机颜色
Private Function getColorTable() As Bgr()
Dim newColors(20) As Bgr
newColors(0) = New Bgr(0, 0, 0) 'background
newColors(1) = New Bgr(128, 0, 0) 'aeroplane
newColors(2) = New Bgr(0, 128, 0) 'bicycle
newColors(3) = New Bgr(128, 128, 0) 'bird
newColors(4) = New Bgr(0, 0, 128) 'boat
newColors(5) = New Bgr(128, 0, 128) 'bottle
newColors(6) = New Bgr(0, 128, 128) 'bus
newColors(7) = New Bgr(128, 128, 128) 'car
newColors(8) = New Bgr(255, 0, 0) 'cat
newColors(9) = New Bgr(0, 255, 0) 'chair
newColors(10) = New Bgr(0, 0, 255) 'cow
newColors(11) = New Bgr(255, 255, 0) 'diningtable
newColors(12) = New Bgr(64, 0, 128) 'dog
newColors(13) = New Bgr(192, 0, 128) 'horse
newColors(14) = New Bgr(64, 128, 128) 'motorbike
newColors(15) = New Bgr(192, 128, 128) 'person
newColors(16) = New Bgr(0, 64, 0) 'pottedplant
newColors(17) = New Bgr(128, 64, 64) 'sheep
newColors(18) = New Bgr(0, 192, 0) 'sofa
newColors(19) = New Bgr(128, 192, 0) 'train
newColors(20) = New Bgr(0, 64, 128) 'tvmonitor
Return newColors
End Function
'显示颜色表及对象名称
'参数1：对象名称的字符串数组
'参数2：bgr颜色数组
Private Sub showColorTable(ByVal names() As String, ByVal colors() As Bgr)
Dim lstmoutV As New List(Of Mat)
For i As Integer = 0 To 20
Dim moutV As New Mat(40, 200, DepthType.Cv8U, 3)
moutV.SetTo(New MCvScalar(colors(i).Blue, colors(i).Green, colors(i).Red))
CvInvoke.PutText(moutV, names(i), New Point(20, 25), FontFace.HersheyTriplex, 0.4, New MCvScalar(255, 255, 255))
lstmoutV.Add(moutV)
Next
Dim mout As New Mat
'垂直方向拼接
CvInvoke.VConcat(lstmoutV.ToArray, mout)
CvInvoke.Imshow("colortable", mout)
End Sub
输出结果如下图所示：
图11-4 FCN分割获得不同对象区域
11.6.2 实例分割 MASK RCNN
Mask RCNN是一种基于Faster R-CNN的实例分割算法，它是一种联合目标检测和语义分割的方法，能够同时检测图像中的对象并得到每个对象的位置、类别和像素级别的分割结果。Mask RCNN在实例分割任务中表现优秀，是目前最先进的实例分割算法之一，被广泛应用于自动驾驶、智能监控等领域。
【代码位置：frmChapter11】Button6_Click、getRadomColor
'实例分割 mask rcnn
Private Sub Button6_Click(sender As Object, e As EventArgs) Handles Button6.Click
'对象分类，object_detection_classes_coco.txt文件提供了90类对象（含background）
Dim classnames() As String
classnames = File.ReadAllLines("C:\learnEmgucv\maskrcnn\object_detection_classes_coco.txt")
'需要测试的图像文件
Dim m As New Mat("C:\learnEmgucv\dnntest.jpg", ImreadModes.Color)
Dim hm As Single = m.Height
Dim wm As Single = m.Width
Dim net As Dnn.Net
net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\maskrcnn\frozen_inference_graph.pb",
"C:\learnEmgucv\maskrcnn\mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"
)
Dim mcopy As New Mat
mcopy = m.Clone
Dim blob As Mat
blob = DnnInvoke.BlobFromImage(mcopy, 1, New Size(500, 500),
New MCvScalar(0， 0， 0), False, False)
net.SetInput(blob)
Dim names(1) As String
names（0） = "detection_out_final"
names（1） = "detection_masks"
'返回的mout包含两个mat
Dim mout As New VectorOfMat
net.Forward(mout, names)
'第一个mat标识返回的置信度候选框，是一个四维数组
Dim moutBox As New Mat
moutBox = mout(0)
'返回维度：
'第1维：1，
'第2维：1，
'第3维：100，100个候选置信矩形框
'第4维：7，0：?;1:对应类别；2：置信度；3-6：候选框位置（源图像百分比）
Dim foutBox(,,,) As Single
foutBox = moutBox.GetData()
'第二个mat标识返回的掩膜，是一个四维数组
Dim moutMask As New Mat
moutMask = mout(1)
'返回维度：
'第1维：100，100个对象对应的100个掩膜
'第2维：90，对象的置信度
'第3维：15，掩膜高度
'第4维：15，掩膜宽度
Dim foutMask(,,,) As Single
foutMask = moutMask.GetData()
Dim maskH As Integer = 15 'foutMask.GetLength(2)
Dim maskW As Integer = 15 'foutMask.GetLength(3)
'新建Mat，用来在这上面绘制掩膜
Dim mbg As New Mat(New Size(m.Width, m.Height), DepthType.Cv8U, 3)
mbg.SetTo(New MCvScalar(0, 0, 0))
For i As Integer = 0 To 100 - 1
'置信度
Dim conf As Single = foutBox(0, 0, i, 2)
'当置信度满足时
If conf > 0.53 Then
'对应检测对象的序号
Dim objID As Integer = foutBox(0, 0, i, 1)
Dim ltX As Single = foutBox(0, 0, i, 3) * wm '左上角X
Dim ltY As Single = foutBox(0, 0, i, 4) * hm '左上角Y
Dim rbX As Single = foutBox(0, 0, i, 5) * wm '右下角X
Dim rbY As Single = foutBox(0, 0, i, 6) * hm '右下角Y
Dim w As Single = rbX - ltX '宽度
Dim h As Single = rbY - ltY '高度
'绘制包围矩形框
CvInvoke.Rectangle(m, New Rectangle(ltX, ltY, w, h), New MCvScalar(0, 0, 255), 1)
'绘制对象名称
CvInvoke.PutText(m, classnames(objID), New Point(ltX, ltY - 10), FontFace.HersheyTriplex, 0.3, New MCvScalar(255, 0, 0))
'开始处理掩膜
Dim mmask As New Mat()
'掩膜大小为15*15，对应foutMask最后两个维度
Dim bmask(14, 14) As Single
For j As Integer = 0 To 14
For k As Integer = 0 To 14
bmask(j, k) = foutMask(i, objID, j, k)
Next
Next
'将数组转为Mat
Dim matrmask As New Matrix(Of Single)(bmask)
mmask = matrmask.Mat
'大小放大与对应包围矩形框一致
CvInvoke.Resize(mmask, mmask, New Drawing.Size(w, h))
'二值化
CvInvoke.Threshold(mmask, mmask, 0.3, 255, ThresholdType.Binary)
'由于本身是CV32F，需要处理为CV8U，才能使用FindContours
mmask.ConvertTo(mmask, DepthType.Cv8U)
'定义关注区域，当修改关注区域时，也就修改了源图像
Dim mRoi As New Mat(mbg, New Rectangle(ltX, ltY, w, h))
'查找轮廓
Dim contours As New VectorOfVectorOfPoint
CvInvoke.FindContours(mmask, contours, Nothing, RetrType.External, ChainApproxMethod.ChainApproxSimple)
'填充轮廓
CvInvoke.DrawContours(mRoi, contours, -1, getRadomColor(), -1)
End If
Next
Dim mresult As New Mat
'加法
CvInvoke.AddWeighted(m, 1, mbg, 0.4, 0, mresult)
ImageBox1.Image = mresult
End Sub
'获得随机颜色
Private Function getRadomColor() As MCvScalar
Dim rd As New Random(Now.Millisecond)
Dim r, g, b As Byte
b = rd.Next(256)
g = rd.Next(256)
r = rd.Next(256)
Return New MCvScalar(b, g, r)
End Function
输出结果如下图所示：
图11-5 Mask RCNN 分割得到对象区域和类别