|
YOLOV4是YOLOV3的改进版，在YOLOV3的基础上结合了非常多的小Tricks。 YOLOV4结构解析 如果大家对YOLOV3比较熟悉的话，应该知道Darknet53的结构，其由一系列残差网络结构构成。在Darknet53中，其存在如下resblock_body模块，其由一次下采样和多次残差结构的堆叠构成，Darknet53便是由resblock_body模块组合而成。 def resblock_body(x, num_filters, num_blocks): x = ZeroPadding2D(((1,0),(1,0)))(x) x = DarknetConv2D_BN_Leaky(num_filters, (3,3), strides=(2,2))(x) for i in range(num_blocks): y = DarknetConv2D_BN_Leaky(num_filters//2, (1,1))(x) y = DarknetConv2D_BN_Leaky(num_filters, (3,3))(y) x = Add()([x,y]) return x
而在YOLOV4中，其对该部分进行了一定的修改。
全部实现代码为： from functools import wrapsfrom tensorflow.keras import backend as Kfrom tensorflow.keras.layers import Conv2D, Add, ZeroPadding2D, UpSampling2D, Concatenate, MaxPooling2D, Layer, LeakyReLU, BatchNormalizationfrom tensorflow.keras.regularizers import l2from utils.utils import composeclass Mish(Layer): def __init__(self, **kwargs): super(Mish, self).__init__(**kwargs) self.supports_masking = True def call(self, inputs): return inputs * K.tanh(K.softplus(inputs)) def get_config(self): config = super(Mish, self).get_config() return config def compute_output_shape(self, input_shape): return input_shape#--------------------------------------------------## 单次卷积#--------------------------------------------------#@wraps(Conv2D)def DarknetConv2D(*args, **kwargs): darknet_conv_kwargs = {'kernel_regularizer': l2(5e-4)} darknet_conv_kwargs['padding'] = 'valid' if kwargs.get('strides')==(2,2) else 'same' darknet_conv_kwargs.update(kwargs) return Conv2D(*args, **darknet_conv_kwargs)#---------------------------------------------------## 卷积块# DarknetConv2D + BatchNormalization + Mish#---------------------------------------------------#def DarknetConv2D_BN_Mish(*args, **kwargs): no_bias_kwargs = {'use_bias': False} no_bias_kwargs.update(kwargs) return compose( DarknetConv2D(*args, **no_bias_kwargs), BatchNormalization(), Mish())#---------------------------------------------------## CSPdarknet的结构块# 存在一个大残差边# 这个大残差边绕过了很多的残差结构#---------------------------------------------------#def resblock_body(x, num_filters, num_blocks, all_narrow=True): # 进行长和宽的压缩 preconv1 = ZeroPadding2D(((1,0),(1,0)))(x) preconv1 = DarknetConv2D_BN_Mish(num_filters, (3,3), strides=(2,2))(preconv1) # 生成一个大的残差边 shortconv = DarknetConv2D_BN_Mish(num_filters//2 if all_narrow else num_filters, (1,1))(preconv1) # 主干部分的卷积 mainconv = DarknetConv2D_BN_Mish(num_filters//2 if all_narrow else num_filters, (1,1))(preconv1) # 1x1卷积对通道数进行整合->3x3卷积提取特征，使用残差结构 for i in range(num_blocks): y = compose( DarknetConv2D_BN_Mish(num_filters//2, (1,1)), DarknetConv2D_BN_Mish(num_filters//2 if all_narrow else num_filters, (3,3)))(mainconv) mainconv = Add()([mainconv,y]) # 1x1卷积后和残差边堆叠 postconv = DarknetConv2D_BN_Mish(num_filters//2 if all_narrow else num_filters, (1,1))(mainconv) route = Concatenate()([postconv, shortconv]) # 最后对通道数进行整合 return DarknetConv2D_BN_Mish(num_filters, (1,1))(route)#---------------------------------------------------## darknet53 的主体部分#---------------------------------------------------#def darknet_body(x): x = DarknetConv2D_BN_Mish(32, (3,3))(x) x = resblock_body(x, 64, 1, False) x = resblock_body(x, 128, 2) x = resblock_body(x, 256, 8) feat1 = x x = resblock_body(x, 512, 8) feat2 = x x = resblock_body(x, 1024, 4) feat3 = x return feat1,feat2,feat3
2、特征金字塔
其可以它能够极大地增加感受野，分离出最显著的上下文特征。 而在YOLOV4当中，其主要是在三个有效特征层上使用了PANet结构。 #---------------------------------------------------## 特征层->最后的输出#---------------------------------------------------#def yolo_body(inputs, num_anchors, num_classes): # 生成darknet53的主干模型 feat1,feat2,feat3 = darknet_body(inputs) P5 = DarknetConv2D_BN_Leaky(512, (1,1))(feat3) P5 = DarknetConv2D_BN_Leaky(1024, (3,3))(P5) P5 = DarknetConv2D_BN_Leaky(512, (1,1))(P5) # 使用了SPP结构，即不同尺度的最大池化后堆叠。 maxpool1 = MaxPooling2D(pool_size=(13,13), strides=(1,1), padding='same')(P5) maxpool2 = MaxPooling2D(pool_size=(9,9), strides=(1,1), padding='same')(P5) maxpool3 = MaxPooling2D(pool_size=(5,5), strides=(1,1), padding='same')(P5) P5 = Concatenate()([maxpool1, maxpool2, maxpool3, P5]) P5 = DarknetConv2D_BN_Leaky(512, (1,1))(P5) P5 = DarknetConv2D_BN_Leaky(1024, (3,3))(P5) P5 = DarknetConv2D_BN_Leaky(512, (1,1))(P5) P5_upsample = compose(DarknetConv2D_BN_Leaky(256, (1,1)), UpSampling2D(2))(P5) P4 = DarknetConv2D_BN_Leaky(256, (1,1))(feat2) P4 = Concatenate()([P4, P5_upsample]) P4 = make_five_convs(P4,256) P4_upsample = compose(DarknetConv2D_BN_Leaky(128, (1,1)), UpSampling2D(2))(P4) P3 = DarknetConv2D_BN_Leaky(128, (1,1))(feat1) P3 = Concatenate()([P3, P4_upsample]) P3 = make_five_convs(P3,128) # 76x76的out P3_output = DarknetConv2D_BN_Leaky(256, (3,3))(P3) P3_output = DarknetConv2D(num_anchors*(num_classes+5), (1,1))(P3_output) P3_downsample = ZeroPadding2D(((1,0),(1,0)))(P3) P3_downsample = DarknetConv2D_BN_Leaky(256, (3,3), strides=(2,2))(P3_downsample) P4 = Concatenate()([P3_downsample, P4]) P4 = make_five_convs(P4,256) # 38x38的out P4_output = DarknetConv2D_BN_Leaky(512, (3,3))(P4) P4_output = DarknetConv2D(num_anchors*(num_classes+5), (1,1))(P4_output) P4_downsample = ZeroPadding2D(((1,0),(1,0)))(P4) P4_downsample = DarknetConv2D_BN_Leaky(512, (3,3), strides=(2,2))(P4_downsample) P5 = Concatenate()([P4_downsample, P5]) P5 = make_five_convs(P5,512) # 19x19的out P5_output = DarknetConv2D_BN_Leaky(1024, (3,3))(P5) P5_output = DarknetConv2D(num_anchors*(num_classes+5), (1,1))(P5_output) return Model(inputs, [P5_output, P4_output, P3_output])
3、YoloHead利用获得到的特征进行预测 2、输出层的shape分别为(19,19,75)，(38,38,75)，(76,76,75)，最后一个维度为75是因为该图是基于voc数据集的，它的类为20种，YoloV4只有针对每一个特征层存在3个先验框，所以最后维度为3x25； 实现代码如下：
4、预测结果的解码 #---------------------------------------------------## 将预测值的每个特征层调成真实值#---------------------------------------------------#def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False): num_anchors = len(anchors) # [1, 1, 1, num_anchors, 2] anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2]) # 获得x，y的网格 # (19, 19, 1, 2) grid_shape = K.shape(feats)[1:3] # height, width grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]), [1, grid_shape[1], 1, 1]) grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]), [grid_shape[0], 1, 1, 1]) grid = K.concatenate([grid_x, grid_y]) grid = K.cast(grid, K.dtype(feats)) # (batch_size,19,19,3,85) feats = K.reshape(feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5]) # 将预测值调成真实值 # box_xy对应框的中心点 # box_wh对应框的宽和高 box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[::-1], K.dtype(feats)) box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats)) box_confidence = K.sigmoid(feats[..., 4:5]) box_class_probs = K.sigmoid(feats[..., 5:]) # 在计算loss的时候返回如下参数 if calc_loss == True: return grid, feats, box_xy, box_wh return box_xy, box_wh, box_confidence, box_class_probs#---------------------------------------------------## 对box进行调整，使其符合真实图片的样子#---------------------------------------------------#def yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape): box_yx = box_xy[..., ::-1] box_hw = box_wh[..., ::-1] input_shape = K.cast(input_shape, K.dtype(box_yx)) image_shape = K.cast(image_shape, K.dtype(box_yx)) new_shape = K.round(image_shape * K.min(input_shape/image_shape)) offset = (input_shape-new_shape)/2./input_shape scale = input_shape/new_shape box_yx = (box_yx - offset) * scale box_hw *= scale box_mins = box_yx - (box_hw / 2.) box_maxes = box_yx + (box_hw / 2.) boxes = K.concatenate([ box_mins[..., 0:1], # y_min box_mins[..., 1:2], # x_min box_maxes[..., 0:1], # y_max box_maxes[..., 1:2] # x_max ]) boxes *= K.concatenate([image_shape, image_shape]) return boxes#---------------------------------------------------## 获取每个box和它的得分#---------------------------------------------------#def yolo_boxes_and_scores(feats, anchors, num_classes, input_shape, image_shape): # 将预测值调成真实值 # box_xy对应框的中心点 # box_wh对应框的宽和高 # -1,19,19,3,2; -1,19,19,3,2; -1,19,19,3,1; -1,19,19,3,80 box_xy, box_wh, box_confidence, box_class_probs = yolo_head(feats, anchors, num_classes, input_shape) # 将box_xy、和box_wh调节成y_min,y_max,xmin,xmax boxes = yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape) # 获得得分和box boxes = K.reshape(boxes, [-1, 4]) box_scores = box_confidence * box_class_probs box_scores = K.reshape(box_scores, [-1, num_classes]) return boxes, box_scores#---------------------------------------------------## 图片预测#---------------------------------------------------#def yolo_eval(yolo_outputs, anchors, num_classes, image_shape, max_boxes=20, score_threshold=.6, iou_threshold=.5): # 获得特征层的数量 num_layers = len(yolo_outputs) # 特征层1对应的anchor是678 # 特征层2对应的anchor是345 # 特征层3对应的anchor是012 anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] input_shape = K.shape(yolo_outputs[0])[1:3] * 32 boxes = [] box_scores = [] # 对每个特征层进行处理 for l in range(num_layers): _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, image_shape) boxes.append(_boxes) box_scores.append(_box_scores) # 将每个特征层的结果进行堆叠 boxes = K.concatenate(boxes, axis=0) box_scores = K.concatenate(box_scores, axis=0) mask = box_scores >= score_threshold max_boxes_tensor = K.constant(max_boxes, dtype='int32') boxes_ = [] scores_ = [] classes_ = [] for c in range(num_classes): # 取出所有box_scores >= score_threshold的框，和成绩 class_boxes = tf.boolean_mask(boxes, mask[:, c]) class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c]) # 非极大抑制，去掉box重合程度高的那一些 nms_index = tf.image.non_max_suppression( class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold) # 获取非极大抑制后的结果 # 下列三个分别是 # 框的位置，得分与种类 class_boxes = K.gather(class_boxes, nms_index) class_box_scores = K.gather(class_box_scores, nms_index) classes = K.ones_like(class_box_scores, 'int32') * c boxes_.append(class_boxes) scores_.append(class_box_scores) classes_.append(classes) boxes_ = K.concatenate(boxes_, axis=0) scores_ = K.concatenate(scores_, axis=0) classes_ = K.concatenate(classes_, axis=0) return boxes_, scores_, classes_
5、在原图上进行绘制 通过第四步，我们可以获得预测框在原图上的位置，而且这些预测框都是经过筛选的。这些筛选后的框可以直接绘制在图片上，就可以获得结果了。 YOLOV4的训练 但是mosaic利用了四张图片，根据论文所说其拥有一个巨大的优点是丰富检测物体的背景！且在BN计算的时候一下子会计算四张图片的数据！ 2、分别对四张图片进行翻转、缩放、色域变化等，并且按照四个方向位置摆好。 3、进行图片的组合和框的组合
b)、Label Smoothing平滑 new_onehot_labels = onehot_labels * (1 - label_smoothing) + label_smoothing / num_classes
当label_smoothing的值为0.01得时候，公式变成如下所示：
其实Label Smoothing平滑就是将标签进行一个平滑，原始的标签是0、1，在平滑后变成0.005(如果是二分类)、0.995，也就是说对分类准确做了一点惩罚，让模型不可以分类的太准确，太准确容易过拟合。 实现代码如下： #---------------------------------------------------## 平滑标签#---------------------------------------------------#def _smooth_labels(y_true, label_smoothing): num_classes = tf.cast(K.shape(y_true)[-1], dtype=K.floatx()) label_smoothing = K.constant(label_smoothing, dtype=K.floatx()) return y_true * (1.0 - label_smoothing) + label_smoothing / num_classes
c)、CIOU
d)、学习率余弦退火衰减 余弦退火衰减法，学习率会先上升再下降，这是退火优化法的思想。（关于什么是退火算法可以百度。） lr_schedule = tf.keras.experimental.CosineDecayRestarts( initial_learning_rate = learning_rate_base, first_decay_steps = 5*epoch_size, t_mul = 1.0, alpha = 1e-2)optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)
2、loss组成 b)、y_pre是什么 c)、y_true是什么。
其输入为： for t, n in enumerate(best_anchor): for l in range(num_layers): if n in anchor_mask[l]: # 计算该目标在第l个特征层所处网格的位置 i = np.floor(true_boxes[b,t,0]*grid_shapes[l][1]).astype('int32') j = np.floor(true_boxes[b,t,1]*grid_shapes[l][0]).astype('int32') # 找到best_anchor索引的索引 k = anchor_mask[l].index(n) c = true_boxes[b,t, 4].astype('int32') # 保存到y_true中 y_true[l][b, j, i, k, 0:4] = true_boxes[b,t, 0:4] y_true[l][b, j, i, k, 4] = 1 y_true[l][b, j, i, k, 5+c] = 1
对于最后输出的y_true而言，只有每个图里每个框最对应的位置有数据，其它的地方都为0。
d)、loss的计算过程 loss值需要对三个特征层进行处理，这里以最小的特征层为例。 其实际上计算的总的loss是三个loss的和，这三个loss分别是： 其实际代码如下，使用yolo_loss就可以获得loss值： #---------------------------------------------------## 平滑标签#---------------------------------------------------#def _smooth_labels(y_true, label_smoothing): num_classes = K.shape(y_true)[-1], label_smoothing = K.constant(label_smoothing, dtype=K.floatx()) return y_true * (1.0 - label_smoothing) + label_smoothing / num_classes#---------------------------------------------------## 将预测值的每个特征层调成真实值#---------------------------------------------------#def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False): num_anchors = len(anchors) # [1, 1, 1, num_anchors, 2] anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2]) # 获得x，y的网格 # (19,19, 1, 2) grid_shape = K.shape(feats)[1:3] # height, width grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]), [1, grid_shape[1], 1, 1]) grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]), [grid_shape[0], 1, 1, 1]) grid = K.concatenate([grid_x, grid_y]) grid = K.cast(grid, K.dtype(feats)) # (batch_size,19,19,3,85) feats = K.reshape(feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5]) # 将预测值调成真实值 # box_xy对应框的中心点 # box_wh对应框的宽和高 box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[::-1], K.dtype(feats)) box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats)) box_confidence = K.sigmoid(feats[..., 4:5]) box_class_probs = K.sigmoid(feats[..., 5:]) # 在计算loss的时候返回如下参数 if calc_loss == True: return grid, feats, box_xy, box_wh return box_xy, box_wh, box_confidence, box_class_probs#---------------------------------------------------## 用于计算每个预测框与真实框的iou#---------------------------------------------------#def box_iou(b1, b2): # 19,19,3,1,4 # 计算左上角的坐标和右下角的坐标 b1 = K.expand_dims(b1, -2) b1_xy = b1[..., :2] b1_wh = b1[..., 2:4] b1_wh_half = b1_wh/2. b1_mins = b1_xy - b1_wh_half b1_maxes = b1_xy + b1_wh_half # 1,n,4 # 计算左上角和右下角的坐标 b2 = K.expand_dims(b2, 0) b2_xy = b2[..., :2] b2_wh = b2[..., 2:4] b2_wh_half = b2_wh/2. b2_mins = b2_xy - b2_wh_half b2_maxes = b2_xy + b2_wh_half # 计算重合面积 intersect_mins = K.maximum(b1_mins, b2_mins) intersect_maxes = K.minimum(b1_maxes, b2_maxes) intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.) intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] b1_area = b1_wh[..., 0] * b1_wh[..., 1] b2_area = b2_wh[..., 0] * b2_wh[..., 1] iou = intersect_area / (b1_area + b2_area - intersect_area) return iou#---------------------------------------------------## loss值计算#---------------------------------------------------#def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, label_smoothing=0.1, print_loss=False): # 一共有三层 num_layers = len(anchors)//3 # 将预测结果和实际ground truth分开，args是[*model_body.output, *y_true] # y_true是一个列表，包含三个特征层，shape分别为(m,13,13,3,85),(m,26,26,3,85),(m,52,52,3,85)。 # yolo_outputs是一个列表，包含三个特征层，shape分别为(m,13,13,255),(m,26,26,255),(m,52,52,255)。 y_true = args[num_layers:] yolo_outputs = args[:num_layers] # 先验框 # 678为142,110, 192,243, 459,401 # 345为36,75, 76,55, 72,146 # 012为12,16, 19,36, 40,28 anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]] # 得到input_shpae为608,608 input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0])) loss = 0 # 取出每一张图片 # m的值就是batch_size m = K.shape(yolo_outputs[0])[0] mf = K.cast(m, K.dtype(yolo_outputs[0])) # y_true是一个列表，包含三个特征层，shape分别为(m,13,13,3,85),(m,26,26,3,85),(m,52,52,3,85)。 # yolo_outputs是一个列表，包含三个特征层，shape分别为(m,13,13,255),(m,26,26,255),(m,52,52,255)。 for l in range(num_layers): # 以第一个特征层(m,13,13,3,85)为例子 # 取出该特征层中存在目标的点的位置。(m,13,13,3,1) object_mask = y_true[l][..., 4:5] # 取出其对应的种类(m,13,13,3,80) true_class_probs = y_true[l][..., 5:] if label_smoothing: true_class_probs = _smooth_labels(true_class_probs, label_smoothing) # 将yolo_outputs的特征层输出进行处理 # grid为网格结构(13,13,1,2)，raw_pred为尚未处理的预测结果(m,13,13,3,85) # 还有解码后的xy，wh，(m,13,13,3,2) grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, calc_loss=True) # 这个是解码后的预测的box的位置 # (m,13,13,3,4) pred_box = K.concatenate([pred_xy, pred_wh]) # 找到负样本群组，第一步是创建一个数组，[] ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True) object_mask_bool = K.cast(object_mask, 'bool') # 对每一张图片计算ignore_mask def loop_body(b, ignore_mask): # 取出第b副图内，真实存在的所有的box的参数 # n,4 true_box = tf.boolean_mask(y_true[l][b,...,0:4], object_mask_bool[b,...,0]) # 计算预测结果与真实情况的iou # pred_box为13,13,3,4 # 计算的结果是每个pred_box和其它所有真实框的iou # 13,13,3,n iou = box_iou(pred_box[b], true_box) # 13,13,3 best_iou = K.max(iou, axis=-1) # 如果某些预测框和真实框的重合程度大于0.5，则忽略。 ignore_mask = ignore_mask.write(b, K.cast(best_iou<ignore_thresh, K.dtype(true_box))) return b+1, ignore_mask # 遍历所有的图片 _, ignore_mask = tf.while_loop(lambda b,*args: b<m, loop_body, [0, ignore_mask]) # 将每幅图的内容压缩，进行处理 ignore_mask = ignore_mask.stack() #(m,13,13,3,1) ignore_mask = K.expand_dims(ignore_mask, -1) box_loss_scale = 2 - y_true[l][...,2:3]*y_true[l][...,3:4] # Calculate ciou loss as location loss raw_true_box = y_true[l][...,0:4] ciou = box_ciou(pred_box, raw_true_box) ciou_loss = object_mask * box_loss_scale * (1 - ciou) ciou_loss = K.sum(ciou_loss) / mf location_loss = ciou_loss # 如果该位置本来有框，那么计算1与置信度的交叉熵 # 如果该位置本来没有框，而且满足best_iou<ignore_thresh，则被认定为负样本 # best_iou<ignore_thresh用于限制负样本数量 confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \ (1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask class_loss = object_mask * K.binary_crossentropy(true_class_probs, raw_pred[...,5:], from_logits=True) confidence_loss = K.sum(confidence_loss) / mf class_loss = K.sum(class_loss) / mf loss += location_loss + confidence_loss + class_loss loss = K.expand_dims(loss, axis=-1) return loss
|
|
|
YOLOV4整体上的检测思路和YOLOV3相比相差并不大，都是使用三个特征层进行分类与回归预测。
CSPnet结构并不算复杂，就是将原来的残差块的堆叠进行了一个拆分，拆成左右两部分：
