You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

metrics.py 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327
  1. # Model validation metrics
  2. import warnings
  3. from pathlib import Path
  4. import math
  5. import matplotlib.pyplot as plt
  6. import numpy as np
  7. import torch
  8. def fitness(x):
  9. # Model fitness as a weighted combination of metrics
  10. w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]
  11. return (x[:, :4] * w).sum(1)
  12. def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir='.', names=()):
  13. """ Compute the average precision, given the recall and precision curves.
  14. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
  15. # Arguments
  16. tp: True positives (nparray, nx1 or nx10).
  17. conf: Objectness value from 0-1 (nparray).
  18. pred_cls: Predicted object classes (nparray).
  19. target_cls: True object classes (nparray).
  20. plot: Plot precision-recall curve at mAP@0.5
  21. save_dir: Plot save directory
  22. # Returns
  23. The average precision as computed in py-faster-rcnn.
  24. """
  25. # Sort by objectness
  26. i = np.argsort(-conf)
  27. tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]
  28. # Find unique classes
  29. unique_classes = np.unique(target_cls)
  30. nc = unique_classes.shape[0] # number of classes, number of detections
  31. # Create Precision-Recall curve and compute AP for each class
  32. px, py = np.linspace(0, 1, 1000), [] # for plotting
  33. ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000))
  34. for ci, c in enumerate(unique_classes):
  35. i = pred_cls == c
  36. n_l = (target_cls == c).sum() # number of labels
  37. n_p = i.sum() # number of predictions
  38. if n_p == 0 or n_l == 0:
  39. continue
  40. else:
  41. # Accumulate FPs and TPs
  42. fpc = (1 - tp[i]).cumsum(0)
  43. tpc = tp[i].cumsum(0)
  44. # Recall
  45. recall = tpc / (n_l + 1e-16) # recall curve
  46. r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases
  47. # Precision
  48. precision = tpc / (tpc + fpc) # precision curve
  49. p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score
  50. # AP from recall-precision curve
  51. for j in range(tp.shape[1]):
  52. ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])
  53. if plot and j == 0:
  54. py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5
  55. # Compute F1 (harmonic mean of precision and recall)
  56. f1 = 2 * p * r / (p + r + 1e-16)
  57. if plot:
  58. plot_pr_curve(px, py, ap, Path(save_dir) / 'PR_curve.png', names)
  59. plot_mc_curve(px, f1, Path(save_dir) / 'F1_curve.png', names, ylabel='F1')
  60. plot_mc_curve(px, p, Path(save_dir) / 'P_curve.png', names, ylabel='Precision')
  61. plot_mc_curve(px, r, Path(save_dir) / 'R_curve.png', names, ylabel='Recall')
  62. i = f1.mean(0).argmax() # max F1 index
  63. return p[:, i], r[:, i], ap, f1[:, i], unique_classes.astype('int32')
  64. def compute_ap(recall, precision):
  65. """ Compute the average precision, given the recall and precision curves
  66. # Arguments
  67. recall: The recall curve (list)
  68. precision: The precision curve (list)
  69. # Returns
  70. Average precision, precision curve, recall curve
  71. """
  72. # Append sentinel values to beginning and end
  73. mrec = np.concatenate(([0.], recall, [recall[-1] + 0.01]))
  74. mpre = np.concatenate(([1.], precision, [0.]))
  75. # Compute the precision envelope
  76. mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))
  77. # Integrate area under curve
  78. method = 'interp' # methods: 'continuous', 'interp'
  79. if method == 'interp':
  80. x = np.linspace(0, 1, 101) # 101-point interp (COCO)
  81. ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate
  82. else: # 'continuous'
  83. i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes
  84. ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve
  85. return ap, mpre, mrec
  86. class ConfusionMatrix:
  87. # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix
  88. def __init__(self, nc, conf=0.25, iou_thres=0.45):
  89. self.matrix = np.zeros((nc + 1, nc + 1))
  90. self.nc = nc # number of classes
  91. self.conf = conf
  92. self.iou_thres = iou_thres
  93. def process_batch(self, detections, labels):
  94. """
  95. Return intersection-over-union (Jaccard index) of boxes.
  96. Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
  97. Arguments:
  98. detections (Array[N, 6]), x1, y1, x2, y2, conf, class
  99. labels (Array[M, 5]), class, x1, y1, x2, y2
  100. Returns:
  101. None, updates confusion matrix accordingly
  102. """
  103. detections = detections[detections[:, 4] > self.conf]
  104. gt_classes = labels[:, 0].int()
  105. detection_classes = detections[:, 5].int()
  106. iou = box_iou(labels[:, 1:], detections[:, :4])
  107. x = torch.where(iou > self.iou_thres)
  108. if x[0].shape[0]:
  109. matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy()
  110. if x[0].shape[0] > 1:
  111. matches = matches[matches[:, 2].argsort()[::-1]]
  112. matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
  113. matches = matches[matches[:, 2].argsort()[::-1]]
  114. matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
  115. else:
  116. matches = np.zeros((0, 3))
  117. n = matches.shape[0] > 0
  118. m0, m1, _ = matches.transpose().astype(np.int16)
  119. for i, gc in enumerate(gt_classes):
  120. j = m0 == i
  121. if n and sum(j) == 1:
  122. self.matrix[detection_classes[m1[j]], gc] += 1 # correct
  123. else:
  124. self.matrix[self.nc, gc] += 1 # background FP
  125. if n:
  126. for i, dc in enumerate(detection_classes):
  127. if not any(m1 == i):
  128. self.matrix[dc, self.nc] += 1 # background FN
  129. def matrix(self):
  130. return self.matrix
  131. def plot(self, normalize=True, save_dir='', names=()):
  132. try:
  133. import seaborn as sn
  134. array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1E-6) if normalize else 1) # normalize columns
  135. array[array < 0.005] = np.nan # don't annotate (would appear as 0.00)
  136. fig = plt.figure(figsize=(12, 9), tight_layout=True)
  137. sn.set(font_scale=1.0 if self.nc < 50 else 0.8) # for label size
  138. labels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels
  139. with warnings.catch_warnings():
  140. warnings.simplefilter('ignore') # suppress empty matrix RuntimeWarning: All-NaN slice encountered
  141. sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True,
  142. xticklabels=names + ['background FP'] if labels else "auto",
  143. yticklabels=names + ['background FN'] if labels else "auto").set_facecolor((1, 1, 1))
  144. fig.axes[0].set_xlabel('True')
  145. fig.axes[0].set_ylabel('Predicted')
  146. fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250)
  147. except Exception as e:
  148. print(f'WARNING: ConfusionMatrix plot failure: {e}')
  149. def print(self):
  150. for i in range(self.nc + 1):
  151. print(' '.join(map(str, self.matrix[i])))
  152. def bbox_iou(box1, box2, x1y1x2y2=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-7):
  153. # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4
  154. box2 = box2.T
  155. # Get the coordinates of bounding boxes
  156. if x1y1x2y2: # x1, y1, x2, y2 = box1
  157. b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]
  158. b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]
  159. else: # transform from xywh to xyxy
  160. b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2
  161. b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2
  162. b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2
  163. b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2
  164. # Intersection area
  165. inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \
  166. (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)
  167. # Union Area
  168. w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps
  169. w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps
  170. union = w1 * h1 + w2 * h2 - inter + eps
  171. iou = inter / union
  172. if GIoU or DIoU or CIoU:
  173. cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width
  174. ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height
  175. if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1
  176. c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared
  177. rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 +
  178. (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared
  179. if DIoU:
  180. return iou - rho2 / c2 # DIoU
  181. elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47
  182. v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)
  183. with torch.no_grad():
  184. alpha = v / (v - iou + (1 + eps))
  185. return iou - (rho2 / c2 + v * alpha) # CIoU
  186. else: # GIoU https://arxiv.org/pdf/1902.09630.pdf
  187. c_area = cw * ch + eps # convex area
  188. return iou - (c_area - union) / c_area # GIoU
  189. else:
  190. return iou # IoU
  191. def box_iou(box1, box2):
  192. # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py
  193. """
  194. Return intersection-over-union (Jaccard index) of boxes.
  195. Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
  196. Arguments:
  197. box1 (Tensor[N, 4])
  198. box2 (Tensor[M, 4])
  199. Returns:
  200. iou (Tensor[N, M]): the NxM matrix containing the pairwise
  201. IoU values for every element in boxes1 and boxes2
  202. """
  203. def box_area(box):
  204. # box = 4xn
  205. return (box[2] - box[0]) * (box[3] - box[1])
  206. area1 = box_area(box1.T)
  207. area2 = box_area(box2.T)
  208. # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)
  209. inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)
  210. return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter)
  211. def bbox_ioa(box1, box2, eps=1E-7):
  212. """ Returns the intersection over box2 area given box1, box2. Boxes are x1y1x2y2
  213. box1: np.array of shape(4)
  214. box2: np.array of shape(nx4)
  215. returns: np.array of shape(n)
  216. """
  217. box2 = box2.transpose()
  218. # Get the coordinates of bounding boxes
  219. b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]
  220. b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]
  221. # Intersection area
  222. inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * \
  223. (np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)).clip(0)
  224. # box2 area
  225. box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + eps
  226. # Intersection over box2 area
  227. return inter_area / box2_area
  228. def wh_iou(wh1, wh2):
  229. # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2
  230. wh1 = wh1[:, None] # [N,1,2]
  231. wh2 = wh2[None] # [1,M,2]
  232. inter = torch.min(wh1, wh2).prod(2) # [N,M]
  233. return inter / (wh1.prod(2) + wh2.prod(2) - inter) # iou = inter / (area1 + area2 - inter)
  234. # Plots ----------------------------------------------------------------------------------------------------------------
  235. def plot_pr_curve(px, py, ap, save_dir='pr_curve.png', names=()):
  236. # Precision-recall curve
  237. fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)
  238. py = np.stack(py, axis=1)
  239. if 0 < len(names) < 21: # display per-class legend if < 21 classes
  240. for i, y in enumerate(py.T):
  241. ax.plot(px, y, linewidth=1, label=f'{names[i]} {ap[i, 0]:.3f}') # plot(recall, precision)
  242. else:
  243. ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision)
  244. ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean())
  245. ax.set_xlabel('Recall')
  246. ax.set_ylabel('Precision')
  247. ax.set_xlim(0, 1)
  248. ax.set_ylim(0, 1)
  249. plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left")
  250. fig.savefig(Path(save_dir), dpi=250)
  251. def plot_mc_curve(px, py, save_dir='mc_curve.png', names=(), xlabel='Confidence', ylabel='Metric'):
  252. # Metric-confidence curve
  253. fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)
  254. if 0 < len(names) < 21: # display per-class legend if < 21 classes
  255. for i, y in enumerate(py):
  256. ax.plot(px, y, linewidth=1, label=f'{names[i]}') # plot(confidence, metric)
  257. else:
  258. ax.plot(px, py.T, linewidth=1, color='grey') # plot(confidence, metric)
  259. y = py.mean(0)
  260. ax.plot(px, y, linewidth=3, color='blue', label=f'all classes {y.max():.2f} at {px[y.argmax()]:.3f}')
  261. ax.set_xlabel(xlabel)
  262. ax.set_ylabel(ylabel)
  263. ax.set_xlim(0, 1)
  264. ax.set_ylim(0, 1)
  265. plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left")
  266. fig.savefig(Path(save_dir), dpi=250)