您最多选择25个主题 主题必须以字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符

444 行
21KB

  1. import argparse
  2. import torch.distributed as dist
  3. import torch.nn.functional as F
  4. import torch.optim as optim
  5. import torch.optim.lr_scheduler as lr_scheduler
  6. import yaml
  7. from torch.utils.tensorboard import SummaryWriter
  8. import test # import test.py to get mAP after each epoch
  9. from models.yolo import Model
  10. from utils.datasets import *
  11. from utils.utils import *
  12. mixed_precision = True
  13. try: # Mixed precision training https://github.com/NVIDIA/apex
  14. from apex import amp
  15. except:
  16. print('Apex recommended for faster mixed precision training: https://github.com/NVIDIA/apex')
  17. mixed_precision = False # not installed
  18. wdir = 'weights' + os.sep # weights dir
  19. last = wdir + 'last.pt'
  20. best = wdir + 'best.pt'
  21. results_file = 'results.txt'
  22. # Hyperparameters
  23. hyp = {'lr0': 0.01, # initial learning rate (SGD=1E-2, Adam=1E-3)
  24. 'momentum': 0.937, # SGD momentum
  25. 'weight_decay': 5e-4, # optimizer weight decay
  26. 'giou': 0.05, # giou loss gain
  27. 'cls': 0.58, # cls loss gain
  28. 'cls_pw': 1.0, # cls BCELoss positive_weight
  29. 'obj': 1.0, # obj loss gain (*=img_size/320 if img_size != 320)
  30. 'obj_pw': 1.0, # obj BCELoss positive_weight
  31. 'iou_t': 0.20, # iou training threshold
  32. 'anchor_t': 4.0, # anchor-multiple threshold
  33. 'fl_gamma': 0.0, # focal loss gamma (efficientDet default is gamma=1.5)
  34. 'hsv_h': 0.014, # image HSV-Hue augmentation (fraction)
  35. 'hsv_s': 0.68, # image HSV-Saturation augmentation (fraction)
  36. 'hsv_v': 0.36, # image HSV-Value augmentation (fraction)
  37. 'degrees': 0.0, # image rotation (+/- deg)
  38. 'translate': 0.0, # image translation (+/- fraction)
  39. 'scale': 0.5, # image scale (+/- gain)
  40. 'shear': 0.0} # image shear (+/- deg)
  41. print(hyp)
  42. # Overwrite hyp with hyp*.txt (optional)
  43. f = glob.glob('hyp*.txt')
  44. if f:
  45. print('Using %s' % f[0])
  46. for k, v in zip(hyp.keys(), np.loadtxt(f[0])):
  47. hyp[k] = v
  48. # Print focal loss if gamma > 0
  49. if hyp['fl_gamma']:
  50. print('Using FocalLoss(gamma=%g)' % hyp['fl_gamma'])
  51. def train(hyp):
  52. epochs = opt.epochs # 300
  53. batch_size = opt.batch_size # 64
  54. weights = opt.weights # initial training weights
  55. # Configure
  56. init_seeds()
  57. with open(opt.data) as f:
  58. data_dict = yaml.load(f, Loader=yaml.FullLoader) # model dict
  59. train_path = data_dict['train']
  60. test_path = data_dict['val']
  61. nc = 1 if opt.single_cls else int(data_dict['nc']) # number of classes
  62. # Remove previous results
  63. for f in glob.glob('*_batch*.jpg') + glob.glob(results_file):
  64. os.remove(f)
  65. # Create model
  66. model = Model(opt.cfg).to(device)
  67. # Image sizes
  68. gs = int(max(model.stride)) # grid size (max stride)
  69. if any(x % gs != 0 for x in opt.img_size):
  70. print('WARNING: --img-size %g,%g must be multiple of %s max stride %g' % (*opt.img_size, opt.cfg, gs))
  71. imgsz, imgsz_test = [make_divisible(x, gs) for x in opt.img_size] # image sizes (train, test)
  72. # Optimizer
  73. nbs = 64 # nominal batch size
  74. accumulate = max(round(nbs / batch_size), 1) # accumulate loss before optimizing
  75. hyp['weight_decay'] *= batch_size * accumulate / nbs # scale weight_decay
  76. pg0, pg1, pg2 = [], [], [] # optimizer parameter groups
  77. for k, v in model.named_parameters():
  78. if v.requires_grad:
  79. if '.bias' in k:
  80. pg2.append(v) # biases
  81. elif '.weight' in k and '.bn' not in k:
  82. pg1.append(v) # apply weight decay
  83. else:
  84. pg0.append(v) # all else
  85. optimizer = optim.Adam(pg0, lr=hyp['lr0']) if opt.adam else \
  86. optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True)
  87. optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay
  88. optimizer.add_param_group({'params': pg2}) # add pg2 (biases)
  89. print('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0)))
  90. del pg0, pg1, pg2
  91. # Load Model
  92. google_utils.attempt_download(weights)
  93. start_epoch, best_fitness = 0, 0.0
  94. if weights.endswith('.pt'): # pytorch format
  95. ckpt = torch.load(weights, map_location=device) # load checkpoint
  96. # load model
  97. try:
  98. ckpt['model'] = \
  99. {k: v for k, v in ckpt['model'].state_dict().items() if model.state_dict()[k].numel() == v.numel()}
  100. model.load_state_dict(ckpt['model'], strict=False)
  101. except KeyError as e:
  102. s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s." \
  103. % (opt.weights, opt.cfg, opt.weights)
  104. raise KeyError(s) from e
  105. # load optimizer
  106. if ckpt['optimizer'] is not None:
  107. optimizer.load_state_dict(ckpt['optimizer'])
  108. best_fitness = ckpt['best_fitness']
  109. # load results
  110. if ckpt.get('training_results') is not None:
  111. with open(results_file, 'w') as file:
  112. file.write(ckpt['training_results']) # write results.txt
  113. start_epoch = ckpt['epoch'] + 1
  114. del ckpt
  115. # Mixed precision training https://github.com/NVIDIA/apex
  116. if mixed_precision:
  117. model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)
  118. # Scheduler https://arxiv.org/pdf/1812.01187.pdf
  119. lf = lambda x: (((1 + math.cos(x * math.pi / epochs)) / 2) ** 1.0) * 0.9 + 0.1 # cosine
  120. scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
  121. scheduler.last_epoch = start_epoch - 1 # do not move
  122. # https://discuss.pytorch.org/t/a-problem-occured-when-resuming-an-optimizer/28822
  123. # plot_lr_scheduler(optimizer, scheduler, epochs)
  124. # Initialize distributed training
  125. if device.type != 'cpu' and torch.cuda.device_count() > 1 and torch.distributed.is_available():
  126. dist.init_process_group(backend='nccl', # distributed backend
  127. init_method='tcp://127.0.0.1:9999', # init method
  128. world_size=1, # number of nodes
  129. rank=0) # node rank
  130. model = torch.nn.parallel.DistributedDataParallel(model)
  131. # Dataset
  132. dataset = LoadImagesAndLabels(train_path, imgsz, batch_size,
  133. augment=True,
  134. hyp=hyp, # augmentation hyperparameters
  135. rect=opt.rect, # rectangular training
  136. cache_images=opt.cache_images,
  137. single_cls=opt.single_cls)
  138. mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class
  139. assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Correct your labels or your model.' % (mlc, nc, opt.cfg)
  140. # Dataloader
  141. batch_size = min(batch_size, len(dataset))
  142. nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]) # number of workers
  143. dataloader = torch.utils.data.DataLoader(dataset,
  144. batch_size=batch_size,
  145. num_workers=nw,
  146. shuffle=not opt.rect, # Shuffle=True unless rectangular training is used
  147. pin_memory=True,
  148. collate_fn=dataset.collate_fn)
  149. # Testloader
  150. testloader = torch.utils.data.DataLoader(LoadImagesAndLabels(test_path, imgsz_test, batch_size,
  151. hyp=hyp,
  152. rect=True,
  153. cache_images=opt.cache_images,
  154. single_cls=opt.single_cls),
  155. batch_size=batch_size,
  156. num_workers=nw,
  157. pin_memory=True,
  158. collate_fn=dataset.collate_fn)
  159. # Model parameters
  160. hyp['cls'] *= nc / 80. # scale coco-tuned hyp['cls'] to current dataset
  161. model.nc = nc # attach number of classes to model
  162. model.hyp = hyp # attach hyperparameters to model
  163. model.gr = 1.0 # giou loss ratio (obj_loss = 1.0 or giou)
  164. model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) # attach class weights
  165. model.names = data_dict['names']
  166. # class frequency
  167. labels = np.concatenate(dataset.labels, 0)
  168. c = torch.tensor(labels[:, 0]) # classes
  169. # cf = torch.bincount(c.long(), minlength=nc) + 1.
  170. # model._initialize_biases(cf.to(device))
  171. plot_labels(labels)
  172. tb_writer.add_histogram('classes', c, 0)
  173. # Exponential moving average
  174. ema = torch_utils.ModelEMA(model)
  175. # Start training
  176. t0 = time.time()
  177. nb = len(dataloader) # number of batches
  178. n_burn = max(3 * nb, 1e3) # burn-in iterations, max(3 epochs, 1k iterations)
  179. maps = np.zeros(nc) # mAP per class
  180. results = (0, 0, 0, 0, 0, 0, 0) # 'P', 'R', 'mAP', 'F1', 'val GIoU', 'val Objectness', 'val Classification'
  181. print('Image sizes %g train, %g test' % (imgsz, imgsz_test))
  182. print('Using %g dataloader workers' % nw)
  183. print('Starting training for %g epochs...' % epochs)
  184. # torch.autograd.set_detect_anomaly(True)
  185. for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------
  186. model.train()
  187. # Update image weights (optional)
  188. if dataset.image_weights:
  189. w = model.class_weights.cpu().numpy() * (1 - maps) ** 2 # class weights
  190. image_weights = labels_to_image_weights(dataset.labels, nc=nc, class_weights=w)
  191. dataset.indices = random.choices(range(dataset.n), weights=image_weights, k=dataset.n) # rand weighted idx
  192. mloss = torch.zeros(4, device=device) # mean losses
  193. print(('\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'GIoU', 'obj', 'cls', 'total', 'targets', 'img_size'))
  194. pbar = tqdm(enumerate(dataloader), total=nb) # progress bar
  195. for i, (imgs, targets, paths, _) in pbar: # batch -------------------------------------------------------------
  196. ni = i + nb * epoch # number integrated batches (since train start)
  197. imgs = imgs.to(device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0
  198. # Burn-in
  199. if ni <= n_burn:
  200. xi = [0, n_burn] # x interp
  201. # model.gr = np.interp(ni, xi, [0.0, 1.0]) # giou loss ratio (obj_loss = 1.0 or giou)
  202. accumulate = max(1, np.interp(ni, xi, [1, nbs / batch_size]).round())
  203. for j, x in enumerate(optimizer.param_groups):
  204. # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
  205. x['lr'] = np.interp(ni, xi, [0.1 if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])
  206. if 'momentum' in x:
  207. x['momentum'] = np.interp(ni, xi, [0.9, hyp['momentum']])
  208. # Multi-scale
  209. if True:
  210. imgsz = random.randrange(640, 640 + gs) // gs * gs
  211. sf = imgsz / max(imgs.shape[2:]) # scale factor
  212. if sf != 1:
  213. ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple)
  214. imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False)
  215. # Forward
  216. pred = model(imgs)
  217. # Loss
  218. loss, loss_items = compute_loss(pred, targets.to(device), model)
  219. if not torch.isfinite(loss):
  220. print('WARNING: non-finite loss, ending training ', loss_items)
  221. return results
  222. # Backward
  223. if mixed_precision:
  224. with amp.scale_loss(loss, optimizer) as scaled_loss:
  225. scaled_loss.backward()
  226. else:
  227. loss.backward()
  228. # Optimize
  229. if ni % accumulate == 0:
  230. optimizer.step()
  231. optimizer.zero_grad()
  232. ema.update(model)
  233. # Print
  234. mloss = (mloss * i + loss_items) / (i + 1) # update mean losses
  235. mem = '%.3gG' % (torch.cuda.memory_cached() / 1E9 if torch.cuda.is_available() else 0) # (GB)
  236. s = ('%10s' * 2 + '%10.4g' * 6) % ('%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgsz)
  237. pbar.set_description(s)
  238. # Plot
  239. if ni < 3:
  240. f = 'train_batch%g.jpg' % i # filename
  241. res = plot_images(images=imgs, targets=targets, paths=paths, fname=f)
  242. if tb_writer:
  243. tb_writer.add_image(f, res, dataformats='HWC', global_step=epoch)
  244. # tb_writer.add_graph(model, imgs) # add model to tensorboard
  245. # end batch ------------------------------------------------------------------------------------------------
  246. # Scheduler
  247. scheduler.step()
  248. # mAP
  249. ema.update_attr(model)
  250. final_epoch = epoch + 1 == epochs
  251. if not opt.notest or final_epoch: # Calculate mAP
  252. results, maps, times = test.test(opt.data,
  253. batch_size=batch_size,
  254. imgsz=imgsz_test,
  255. save_json=final_epoch and opt.data.endswith(os.sep + 'coco.yaml'),
  256. model=ema.ema,
  257. single_cls=opt.single_cls,
  258. dataloader=testloader,
  259. multi_label=ni > n_burn)
  260. # Write
  261. with open(results_file, 'a') as f:
  262. f.write(s + '%10.4g' * 7 % results + '\n') # P, R, mAP, F1, test_losses=(GIoU, obj, cls)
  263. if len(opt.name) and opt.bucket:
  264. os.system('gsutil cp results.txt gs://%s/results/results%s.txt' % (opt.bucket, opt.name))
  265. # Tensorboard
  266. if tb_writer:
  267. tags = ['train/giou_loss', 'train/obj_loss', 'train/cls_loss',
  268. 'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/F1',
  269. 'val/giou_loss', 'val/obj_loss', 'val/cls_loss']
  270. for x, tag in zip(list(mloss[:-1]) + list(results), tags):
  271. tb_writer.add_scalar(tag, x, epoch)
  272. # Update best mAP
  273. fi = fitness(np.array(results).reshape(1, -1)) # fitness_i = weighted combination of [P, R, mAP, F1]
  274. if fi > best_fitness:
  275. best_fitness = fi
  276. # Save model
  277. save = (not opt.nosave) or (final_epoch and not opt.evolve)
  278. if save:
  279. with open(results_file, 'r') as f: # create checkpoint
  280. ckpt = {'epoch': epoch,
  281. 'best_fitness': best_fitness,
  282. 'training_results': f.read(),
  283. 'model': ema.ema.module if hasattr(model, 'module') else ema.ema,
  284. 'optimizer': None if final_epoch else optimizer.state_dict()}
  285. # Save last, best and delete
  286. torch.save(ckpt, last)
  287. if (best_fitness == fi) and not final_epoch:
  288. torch.save(ckpt, best)
  289. del ckpt
  290. # end epoch ----------------------------------------------------------------------------------------------------
  291. # end training
  292. n = opt.name
  293. if len(n):
  294. n = '_' + n if not n.isnumeric() else n
  295. fresults, flast, fbest = 'results%s.txt' % n, wdir + 'last%s.pt' % n, wdir + 'best%s.pt' % n
  296. for f1, f2 in zip([wdir + 'last.pt', wdir + 'best.pt', 'results.txt'], [flast, fbest, fresults]):
  297. if os.path.exists(f1):
  298. os.rename(f1, f2) # rename
  299. ispt = f2.endswith('.pt') # is *.pt
  300. strip_optimizer(f2) if ispt else None # strip optimizer
  301. os.system('gsutil cp %s gs://%s/weights' % (f2, opt.bucket)) if opt.bucket and ispt else None # upload
  302. if not opt.evolve:
  303. plot_results() # save as results.png
  304. print('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600))
  305. dist.destroy_process_group() if torch.cuda.device_count() > 1 else None
  306. torch.cuda.empty_cache()
  307. return results
  308. if __name__ == '__main__':
  309. parser = argparse.ArgumentParser()
  310. parser.add_argument('--epochs', type=int, default=300)
  311. parser.add_argument('--batch-size', type=int, default=16)
  312. parser.add_argument('--cfg', type=str, default='models/yolov5s.yaml', help='*.cfg path')
  313. parser.add_argument('--data', type=str, default='data/coco128.yaml', help='*.data path')
  314. parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='train,test sizes')
  315. parser.add_argument('--rect', action='store_true', help='rectangular training')
  316. parser.add_argument('--resume', action='store_true', help='resume training from last.pt')
  317. parser.add_argument('--nosave', action='store_true', help='only save final checkpoint')
  318. parser.add_argument('--notest', action='store_true', help='only test final epoch')
  319. parser.add_argument('--evolve', action='store_true', help='evolve hyperparameters')
  320. parser.add_argument('--bucket', type=str, default='', help='gsutil bucket')
  321. parser.add_argument('--cache-images', action='store_true', help='cache images for faster training')
  322. parser.add_argument('--weights', type=str, default='', help='initial weights path')
  323. parser.add_argument('--name', default='', help='renames results.txt to results_name.txt if supplied')
  324. parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
  325. parser.add_argument('--adam', action='store_true', help='use adam optimizer')
  326. parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
  327. opt = parser.parse_args()
  328. opt.weights = last if opt.resume else opt.weights
  329. opt.cfg = glob.glob('./**/' + opt.cfg, recursive=True)[0] # find file
  330. opt.data = glob.glob('./**/' + opt.data, recursive=True)[0] # find file
  331. print(opt)
  332. opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test)
  333. device = torch_utils.select_device(opt.device, apex=mixed_precision, batch_size=opt.batch_size)
  334. # check_git_status()
  335. if device.type == 'cpu':
  336. mixed_precision = False
  337. # Train
  338. if not opt.evolve:
  339. tb_writer = SummaryWriter(comment=opt.name)
  340. print('Start Tensorboard with "tensorboard --logdir=runs", view at http://localhost:6006/')
  341. train(hyp)
  342. # Evolve hyperparameters (optional)
  343. else:
  344. tb_writer = None
  345. opt.notest, opt.nosave = True, True # only test/save final epoch
  346. if opt.bucket:
  347. os.system('gsutil cp gs://%s/evolve.txt .' % opt.bucket) # download evolve.txt if exists
  348. for _ in range(10): # generations to evolve
  349. if os.path.exists('evolve.txt'): # if evolve.txt exists: select best hyps and mutate
  350. # Select parent(s)
  351. parent = 'single' # parent selection method: 'single' or 'weighted'
  352. x = np.loadtxt('evolve.txt', ndmin=2)
  353. n = min(5, len(x)) # number of previous results to consider
  354. x = x[np.argsort(-fitness(x))][:n] # top n mutations
  355. w = fitness(x) - fitness(x).min() # weights
  356. if parent == 'single' or len(x) == 1:
  357. # x = x[random.randint(0, n - 1)] # random selection
  358. x = x[random.choices(range(n), weights=w)[0]] # weighted selection
  359. elif parent == 'weighted':
  360. x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination
  361. # Mutate
  362. mp, s = 0.9, 0.2 # mutation probability, sigma
  363. npr = np.random
  364. npr.seed(int(time.time()))
  365. g = np.array([1, 1, 1, 1, 1, 1, 1, 0, .1, 1, 0, 1, 1, 1, 1, 1, 1, 1]) # gains
  366. ng = len(g)
  367. v = np.ones(ng)
  368. while all(v == 1): # mutate until a change occurs (prevent duplicates)
  369. v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0)
  370. for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300)
  371. hyp[k] = x[i + 7] * v[i] # mutate
  372. # Clip to limits
  373. keys = ['lr0', 'iou_t', 'momentum', 'weight_decay', 'hsv_s', 'hsv_v', 'translate', 'scale', 'fl_gamma']
  374. limits = [(1e-5, 1e-2), (0.00, 0.70), (0.60, 0.98), (0, 0.001), (0, .9), (0, .9), (0, .9), (0, .9), (0, 3)]
  375. for k, v in zip(keys, limits):
  376. hyp[k] = np.clip(hyp[k], v[0], v[1])
  377. # Train mutation
  378. results = train(hyp.copy())
  379. # Write mutation results
  380. print_mutation(hyp, results, opt.bucket)
  381. # Plot results
  382. # plot_evolution_results(hyp)