You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

202 lines
7.8KB

  1. # Model validation metrics
  2. from pathlib import Path
  3. import matplotlib.pyplot as plt
  4. import numpy as np
  5. import torch
  6. from . import general
  7. def fitness(x):
  8. # Model fitness as a weighted combination of metrics
  9. w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]
  10. return (x[:, :4] * w).sum(1)
  11. def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir='precision-recall_curve.png', names=[]):
  12. """ Compute the average precision, given the recall and precision curves.
  13. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
  14. # Arguments
  15. tp: True positives (nparray, nx1 or nx10).
  16. conf: Objectness value from 0-1 (nparray).
  17. pred_cls: Predicted object classes (nparray).
  18. target_cls: True object classes (nparray).
  19. plot: Plot precision-recall curve at mAP@0.5
  20. save_dir: Plot save directory
  21. # Returns
  22. The average precision as computed in py-faster-rcnn.
  23. """
  24. # Sort by objectness
  25. i = np.argsort(-conf)
  26. tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]
  27. # Find unique classes
  28. unique_classes = np.unique(target_cls)
  29. # Create Precision-Recall curve and compute AP for each class
  30. px, py = np.linspace(0, 1, 1000), [] # for plotting
  31. pr_score = 0.1 # score to evaluate P and R https://github.com/ultralytics/yolov3/issues/898
  32. s = [unique_classes.shape[0], tp.shape[1]] # number class, number iou thresholds (i.e. 10 for mAP0.5...0.95)
  33. ap, p, r = np.zeros(s), np.zeros(s), np.zeros(s)
  34. for ci, c in enumerate(unique_classes):
  35. i = pred_cls == c
  36. n_l = (target_cls == c).sum() # number of labels
  37. n_p = i.sum() # number of predictions
  38. if n_p == 0 or n_l == 0:
  39. continue
  40. else:
  41. # Accumulate FPs and TPs
  42. fpc = (1 - tp[i]).cumsum(0)
  43. tpc = tp[i].cumsum(0)
  44. # Recall
  45. recall = tpc / (n_l + 1e-16) # recall curve
  46. r[ci] = np.interp(-pr_score, -conf[i], recall[:, 0]) # r at pr_score, negative x, xp because xp decreases
  47. # Precision
  48. precision = tpc / (tpc + fpc) # precision curve
  49. p[ci] = np.interp(-pr_score, -conf[i], precision[:, 0]) # p at pr_score
  50. # AP from recall-precision curve
  51. for j in range(tp.shape[1]):
  52. ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])
  53. if plot and (j == 0):
  54. py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5
  55. # Compute F1 score (harmonic mean of precision and recall)
  56. f1 = 2 * p * r / (p + r + 1e-16)
  57. if plot:
  58. plot_pr_curve(px, py, ap, save_dir, names)
  59. return p, r, ap, f1, unique_classes.astype('int32')
  60. def compute_ap(recall, precision):
  61. """ Compute the average precision, given the recall and precision curves.
  62. Source: https://github.com/rbgirshick/py-faster-rcnn.
  63. # Arguments
  64. recall: The recall curve (list).
  65. precision: The precision curve (list).
  66. # Returns
  67. The average precision as computed in py-faster-rcnn.
  68. """
  69. # Append sentinel values to beginning and end
  70. mrec = recall # np.concatenate(([0.], recall, [recall[-1] + 1E-3]))
  71. mpre = precision # np.concatenate(([0.], precision, [0.]))
  72. # Compute the precision envelope
  73. mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))
  74. # Integrate area under curve
  75. method = 'interp' # methods: 'continuous', 'interp'
  76. if method == 'interp':
  77. x = np.linspace(0, 1, 101) # 101-point interp (COCO)
  78. ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate
  79. else: # 'continuous'
  80. i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes
  81. ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve
  82. return ap, mpre, mrec
  83. class ConfusionMatrix:
  84. # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix
  85. def __init__(self, nc, conf=0.25, iou_thres=0.45):
  86. self.matrix = np.zeros((nc + 1, nc + 1))
  87. self.nc = nc # number of classes
  88. self.conf = conf
  89. self.iou_thres = iou_thres
  90. def process_batch(self, detections, labels):
  91. """
  92. Return intersection-over-union (Jaccard index) of boxes.
  93. Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
  94. Arguments:
  95. detections (Array[N, 6]), x1, y1, x2, y2, conf, class
  96. labels (Array[M, 5]), class, x1, y1, x2, y2
  97. Returns:
  98. None, updates confusion matrix accordingly
  99. """
  100. detections = detections[detections[:, 4] > self.conf]
  101. gt_classes = labels[:, 0].int()
  102. detection_classes = detections[:, 5].int()
  103. iou = general.box_iou(labels[:, 1:], detections[:, :4])
  104. x = torch.where(iou > self.iou_thres)
  105. if x[0].shape[0]:
  106. matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy()
  107. if x[0].shape[0] > 1:
  108. matches = matches[matches[:, 2].argsort()[::-1]]
  109. matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
  110. matches = matches[matches[:, 2].argsort()[::-1]]
  111. matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
  112. else:
  113. matches = np.zeros((0, 3))
  114. n = matches.shape[0] > 0
  115. m0, m1, _ = matches.transpose().astype(np.int16)
  116. for i, gc in enumerate(gt_classes):
  117. j = m0 == i
  118. if n and sum(j) == 1:
  119. self.matrix[gc, detection_classes[m1[j]]] += 1 # correct
  120. else:
  121. self.matrix[gc, self.nc] += 1 # background FP
  122. if n:
  123. for i, dc in enumerate(detection_classes):
  124. if not any(m1 == i):
  125. self.matrix[self.nc, dc] += 1 # background FN
  126. def matrix(self):
  127. return self.matrix
  128. def plot(self, save_dir='', names=()):
  129. try:
  130. import seaborn as sn
  131. array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize
  132. array[array < 0.005] = np.nan # don't annotate (would appear as 0.00)
  133. fig = plt.figure(figsize=(12, 9), tight_layout=True)
  134. sn.set(font_scale=1.0 if self.nc < 50 else 0.8) # for label size
  135. labels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels
  136. sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True,
  137. xticklabels=names + ['background FN'] if labels else "auto",
  138. yticklabels=names + ['background FP'] if labels else "auto").set_facecolor((1, 1, 1))
  139. fig.axes[0].set_xlabel('True')
  140. fig.axes[0].set_ylabel('Predicted')
  141. fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250)
  142. except Exception as e:
  143. pass
  144. def print(self):
  145. for i in range(self.nc + 1):
  146. print(' '.join(map(str, self.matrix[i])))
  147. # Plots ----------------------------------------------------------------------------------------------------------------
  148. def plot_pr_curve(px, py, ap, save_dir='.', names=()):
  149. fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)
  150. py = np.stack(py, axis=1)
  151. if 0 < len(names) < 21: # show mAP in legend if < 10 classes
  152. for i, y in enumerate(py.T):
  153. ax.plot(px, y, linewidth=1, label=f'{names[i]} %.3f' % ap[i, 0]) # plot(recall, precision)
  154. else:
  155. ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision)
  156. ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean())
  157. ax.set_xlabel('Recall')
  158. ax.set_ylabel('Precision')
  159. ax.set_xlim(0, 1)
  160. ax.set_ylim(0, 1)
  161. plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left")
  162. fig.savefig(Path(save_dir) / 'precision_recall_curve.png', dpi=250)