You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. # Model validation metrics
  2. import warnings
  3. from pathlib import Path
  4. import matplotlib.pyplot as plt
  5. import numpy as np
  6. import torch
  7. from . import general
  8. def fitness(x):
  9. # Model fitness as a weighted combination of metrics
  10. w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]
  11. return (x[:, :4] * w).sum(1)
  12. def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir='.', names=()):
  13. """ Compute the average precision, given the recall and precision curves.
  14. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
  15. # Arguments
  16. tp: True positives (nparray, nx1 or nx10).
  17. conf: Objectness value from 0-1 (nparray).
  18. pred_cls: Predicted object classes (nparray).
  19. target_cls: True object classes (nparray).
  20. plot: Plot precision-recall curve at mAP@0.5
  21. save_dir: Plot save directory
  22. # Returns
  23. The average precision as computed in py-faster-rcnn.
  24. """
  25. # Sort by objectness
  26. i = np.argsort(-conf)
  27. tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]
  28. # Find unique classes
  29. unique_classes = np.unique(target_cls)
  30. nc = unique_classes.shape[0] # number of classes, number of detections
  31. # Create Precision-Recall curve and compute AP for each class
  32. px, py = np.linspace(0, 1, 1000), [] # for plotting
  33. ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000))
  34. for ci, c in enumerate(unique_classes):
  35. i = pred_cls == c
  36. n_l = (target_cls == c).sum() # number of labels
  37. n_p = i.sum() # number of predictions
  38. if n_p == 0 or n_l == 0:
  39. continue
  40. else:
  41. # Accumulate FPs and TPs
  42. fpc = (1 - tp[i]).cumsum(0)
  43. tpc = tp[i].cumsum(0)
  44. # Recall
  45. recall = tpc / (n_l + 1e-16) # recall curve
  46. r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases
  47. # Precision
  48. precision = tpc / (tpc + fpc) # precision curve
  49. p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score
  50. # AP from recall-precision curve
  51. for j in range(tp.shape[1]):
  52. ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])
  53. if plot and j == 0:
  54. py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5
  55. # Compute F1 (harmonic mean of precision and recall)
  56. f1 = 2 * p * r / (p + r + 1e-16)
  57. if plot:
  58. plot_pr_curve(px, py, ap, Path(save_dir) / 'PR_curve.png', names)
  59. plot_mc_curve(px, f1, Path(save_dir) / 'F1_curve.png', names, ylabel='F1')
  60. plot_mc_curve(px, p, Path(save_dir) / 'P_curve.png', names, ylabel='Precision')
  61. plot_mc_curve(px, r, Path(save_dir) / 'R_curve.png', names, ylabel='Recall')
  62. i = f1.mean(0).argmax() # max F1 index
  63. return p[:, i], r[:, i], ap, f1[:, i], unique_classes.astype('int32')
  64. def compute_ap(recall, precision):
  65. """ Compute the average precision, given the recall and precision curves
  66. # Arguments
  67. recall: The recall curve (list)
  68. precision: The precision curve (list)
  69. # Returns
  70. Average precision, precision curve, recall curve
  71. """
  72. # Append sentinel values to beginning and end
  73. mrec = np.concatenate(([0.], recall, [recall[-1] + 0.01]))
  74. mpre = np.concatenate(([1.], precision, [0.]))
  75. # Compute the precision envelope
  76. mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))
  77. # Integrate area under curve
  78. method = 'interp' # methods: 'continuous', 'interp'
  79. if method == 'interp':
  80. x = np.linspace(0, 1, 101) # 101-point interp (COCO)
  81. ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate
  82. else: # 'continuous'
  83. i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes
  84. ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve
  85. return ap, mpre, mrec
  86. class ConfusionMatrix:
  87. # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix
  88. def __init__(self, nc, conf=0.25, iou_thres=0.45):
  89. self.matrix = np.zeros((nc + 1, nc + 1))
  90. self.nc = nc # number of classes
  91. self.conf = conf
  92. self.iou_thres = iou_thres
  93. def process_batch(self, detections, labels):
  94. """
  95. Return intersection-over-union (Jaccard index) of boxes.
  96. Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
  97. Arguments:
  98. detections (Array[N, 6]), x1, y1, x2, y2, conf, class
  99. labels (Array[M, 5]), class, x1, y1, x2, y2
  100. Returns:
  101. None, updates confusion matrix accordingly
  102. """
  103. detections = detections[detections[:, 4] > self.conf]
  104. gt_classes = labels[:, 0].int()
  105. detection_classes = detections[:, 5].int()
  106. iou = general.box_iou(labels[:, 1:], detections[:, :4])
  107. x = torch.where(iou > self.iou_thres)
  108. if x[0].shape[0]:
  109. matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy()
  110. if x[0].shape[0] > 1:
  111. matches = matches[matches[:, 2].argsort()[::-1]]
  112. matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
  113. matches = matches[matches[:, 2].argsort()[::-1]]
  114. matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
  115. else:
  116. matches = np.zeros((0, 3))
  117. n = matches.shape[0] > 0
  118. m0, m1, _ = matches.transpose().astype(np.int16)
  119. for i, gc in enumerate(gt_classes):
  120. j = m0 == i
  121. if n and sum(j) == 1:
  122. self.matrix[detection_classes[m1[j]], gc] += 1 # correct
  123. else:
  124. self.matrix[self.nc, gc] += 1 # background FP
  125. if n:
  126. for i, dc in enumerate(detection_classes):
  127. if not any(m1 == i):
  128. self.matrix[dc, self.nc] += 1 # background FN
  129. def matrix(self):
  130. return self.matrix
  131. def plot(self, normalize=True, save_dir='', names=()):
  132. try:
  133. import seaborn as sn
  134. array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1E-6) if normalize else 1) # normalize columns
  135. array[array < 0.005] = np.nan # don't annotate (would appear as 0.00)
  136. fig = plt.figure(figsize=(12, 9), tight_layout=True)
  137. sn.set(font_scale=1.0 if self.nc < 50 else 0.8) # for label size
  138. labels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels
  139. with warnings.catch_warnings():
  140. warnings.simplefilter('ignore') # suppress empty matrix RuntimeWarning: All-NaN slice encountered
  141. sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True,
  142. xticklabels=names + ['background FP'] if labels else "auto",
  143. yticklabels=names + ['background FN'] if labels else "auto").set_facecolor((1, 1, 1))
  144. fig.axes[0].set_xlabel('True')
  145. fig.axes[0].set_ylabel('Predicted')
  146. fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250)
  147. except Exception as e:
  148. print(f'WARNING: ConfusionMatrix plot failure: {e}')
  149. def print(self):
  150. for i in range(self.nc + 1):
  151. print(' '.join(map(str, self.matrix[i])))
  152. # Plots ----------------------------------------------------------------------------------------------------------------
  153. def plot_pr_curve(px, py, ap, save_dir='pr_curve.png', names=()):
  154. # Precision-recall curve
  155. fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)
  156. py = np.stack(py, axis=1)
  157. if 0 < len(names) < 21: # display per-class legend if < 21 classes
  158. for i, y in enumerate(py.T):
  159. ax.plot(px, y, linewidth=1, label=f'{names[i]} {ap[i, 0]:.3f}') # plot(recall, precision)
  160. else:
  161. ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision)
  162. ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean())
  163. ax.set_xlabel('Recall')
  164. ax.set_ylabel('Precision')
  165. ax.set_xlim(0, 1)
  166. ax.set_ylim(0, 1)
  167. plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left")
  168. fig.savefig(Path(save_dir), dpi=250)
  169. def plot_mc_curve(px, py, save_dir='mc_curve.png', names=(), xlabel='Confidence', ylabel='Metric'):
  170. # Metric-confidence curve
  171. fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)
  172. if 0 < len(names) < 21: # display per-class legend if < 21 classes
  173. for i, y in enumerate(py):
  174. ax.plot(px, y, linewidth=1, label=f'{names[i]}') # plot(confidence, metric)
  175. else:
  176. ax.plot(px, py.T, linewidth=1, color='grey') # plot(confidence, metric)
  177. y = py.mean(0)
  178. ax.plot(px, y, linewidth=3, color='blue', label=f'all classes {y.max():.2f} at {px[y.argmax()]:.3f}')
  179. ax.set_xlabel(xlabel)
  180. ax.set_ylabel(ylabel)
  181. ax.set_xlim(0, 1)
  182. ax.set_ylim(0, 1)
  183. plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left")
  184. fig.savefig(Path(save_dir), dpi=250)