You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
пре 4 година
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. import argparse
  2. import torch.distributed as dist
  3. import torch.nn.functional as F
  4. import torch.optim as optim
  5. import torch.optim.lr_scheduler as lr_scheduler
  6. import torch.utils.data
  7. from torch.utils.tensorboard import SummaryWriter
  8. import test # import test.py to get mAP after each epoch
  9. from models.yolo import Model
  10. from utils import google_utils
  11. from utils.datasets import *
  12. from utils.utils import *
  13. mixed_precision = True
  14. try: # Mixed precision training https://github.com/NVIDIA/apex
  15. from apex import amp
  16. except:
  17. print('Apex recommended for faster mixed precision training: https://github.com/NVIDIA/apex')
  18. mixed_precision = False # not installed
  19. wdir = 'weights' + os.sep # weights dir
  20. os.makedirs(wdir, exist_ok=True)
  21. last = wdir + 'last.pt'
  22. best = wdir + 'best.pt'
  23. results_file = 'results.txt'
  24. # Hyperparameters
  25. hyp = {'lr0': 0.01, # initial learning rate (SGD=1E-2, Adam=1E-3)
  26. 'momentum': 0.937, # SGD momentum
  27. 'weight_decay': 5e-4, # optimizer weight decay
  28. 'giou': 0.05, # giou loss gain
  29. 'cls': 0.58, # cls loss gain
  30. 'cls_pw': 1.0, # cls BCELoss positive_weight
  31. 'obj': 1.0, # obj loss gain (*=img_size/320 if img_size != 320)
  32. 'obj_pw': 1.0, # obj BCELoss positive_weight
  33. 'iou_t': 0.20, # iou training threshold
  34. 'anchor_t': 4.0, # anchor-multiple threshold
  35. 'fl_gamma': 0.0, # focal loss gamma (efficientDet default is gamma=1.5)
  36. 'hsv_h': 0.014, # image HSV-Hue augmentation (fraction)
  37. 'hsv_s': 0.68, # image HSV-Saturation augmentation (fraction)
  38. 'hsv_v': 0.36, # image HSV-Value augmentation (fraction)
  39. 'degrees': 0.0, # image rotation (+/- deg)
  40. 'translate': 0.0, # image translation (+/- fraction)
  41. 'scale': 0.5, # image scale (+/- gain)
  42. 'shear': 0.0} # image shear (+/- deg)
  43. print(hyp)
  44. # Overwrite hyp with hyp*.txt (optional)
  45. f = glob.glob('hyp*.txt')
  46. if f:
  47. print('Using %s' % f[0])
  48. for k, v in zip(hyp.keys(), np.loadtxt(f[0])):
  49. hyp[k] = v
  50. # Print focal loss if gamma > 0
  51. if hyp['fl_gamma']:
  52. print('Using FocalLoss(gamma=%g)' % hyp['fl_gamma'])
  53. def train(hyp):
  54. epochs = opt.epochs # 300
  55. batch_size = opt.batch_size # 64
  56. weights = opt.weights # initial training weights
  57. # Configure
  58. init_seeds(1)
  59. with open(opt.data) as f:
  60. data_dict = yaml.load(f, Loader=yaml.FullLoader) # model dict
  61. train_path = data_dict['train']
  62. test_path = data_dict['val']
  63. nc = 1 if opt.single_cls else int(data_dict['nc']) # number of classes
  64. # Remove previous results
  65. for f in glob.glob('*_batch*.jpg') + glob.glob(results_file):
  66. os.remove(f)
  67. # Create model
  68. model = Model(opt.cfg).to(device)
  69. assert model.md['nc'] == nc, '%s nc=%g classes but %s nc=%g classes' % (opt.data, nc, opt.cfg, model.md['nc'])
  70. model.names = data_dict['names']
  71. # Image sizes
  72. gs = int(max(model.stride)) # grid size (max stride)
  73. imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size] # verify imgsz are gs-multiples
  74. # Optimizer
  75. nbs = 64 # nominal batch size
  76. accumulate = max(round(nbs / batch_size), 1) # accumulate loss before optimizing
  77. hyp['weight_decay'] *= batch_size * accumulate / nbs # scale weight_decay
  78. pg0, pg1, pg2 = [], [], [] # optimizer parameter groups
  79. for k, v in model.named_parameters():
  80. if v.requires_grad:
  81. if '.bias' in k:
  82. pg2.append(v) # biases
  83. elif '.weight' in k and '.bn' not in k:
  84. pg1.append(v) # apply weight decay
  85. else:
  86. pg0.append(v) # all else
  87. optimizer = optim.Adam(pg0, lr=hyp['lr0']) if opt.adam else \
  88. optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True)
  89. optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay
  90. optimizer.add_param_group({'params': pg2}) # add pg2 (biases)
  91. print('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0)))
  92. del pg0, pg1, pg2
  93. # Load Model
  94. google_utils.attempt_download(weights)
  95. start_epoch, best_fitness = 0, 0.0
  96. if weights.endswith('.pt'): # pytorch format
  97. ckpt = torch.load(weights, map_location=device) # load checkpoint
  98. # load model
  99. try:
  100. ckpt['model'] = {k: v for k, v in ckpt['model'].float().state_dict().items()
  101. if model.state_dict()[k].shape == v.shape} # to FP32, filter
  102. model.load_state_dict(ckpt['model'], strict=False)
  103. except KeyError as e:
  104. s = "%s is not compatible with %s. This may be due to model differences or %s may be out of date. " \
  105. "Please delete or update %s and try again, or use --weights '' to train from scatch." \
  106. % (opt.weights, opt.cfg, opt.weights, opt.weights)
  107. raise KeyError(s) from e
  108. # load optimizer
  109. if ckpt['optimizer'] is not None:
  110. optimizer.load_state_dict(ckpt['optimizer'])
  111. best_fitness = ckpt['best_fitness']
  112. # load results
  113. if ckpt.get('training_results') is not None:
  114. with open(results_file, 'w') as file:
  115. file.write(ckpt['training_results']) # write results.txt
  116. start_epoch = ckpt['epoch'] + 1
  117. del ckpt
  118. # Mixed precision training https://github.com/NVIDIA/apex
  119. if mixed_precision:
  120. model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)
  121. # Scheduler https://arxiv.org/pdf/1812.01187.pdf
  122. lf = lambda x: (((1 + math.cos(x * math.pi / epochs)) / 2) ** 1.0) * 0.9 + 0.1 # cosine
  123. scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
  124. scheduler.last_epoch = start_epoch - 1 # do not move
  125. # https://discuss.pytorch.org/t/a-problem-occured-when-resuming-an-optimizer/28822
  126. # plot_lr_scheduler(optimizer, scheduler, epochs)
  127. # Initialize distributed training
  128. if device.type != 'cpu' and torch.cuda.device_count() > 1 and torch.distributed.is_available():
  129. dist.init_process_group(backend='nccl', # distributed backend
  130. init_method='tcp://127.0.0.1:9999', # init method
  131. world_size=1, # number of nodes
  132. rank=0) # node rank
  133. model = torch.nn.parallel.DistributedDataParallel(model)
  134. # pip install torch==1.4.0+cu100 torchvision==0.5.0+cu100 -f https://download.pytorch.org/whl/torch_stable.html
  135. # Trainloader
  136. dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt,
  137. hyp=hyp, augment=True, cache=opt.cache_images, rect=opt.rect)
  138. mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class
  139. assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Correct your labels or your model.' % (mlc, nc, opt.cfg)
  140. # Testloader
  141. testloader = create_dataloader(test_path, imgsz_test, batch_size, gs, opt,
  142. hyp=hyp, augment=False, cache=opt.cache_images, rect=True)[0]
  143. # Model parameters
  144. hyp['cls'] *= nc / 80. # scale coco-tuned hyp['cls'] to current dataset
  145. model.nc = nc # attach number of classes to model
  146. model.hyp = hyp # attach hyperparameters to model
  147. model.gr = 1.0 # giou loss ratio (obj_loss = 1.0 or giou)
  148. model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) # attach class weights
  149. # Class frequency
  150. labels = np.concatenate(dataset.labels, 0)
  151. c = torch.tensor(labels[:, 0]) # classes
  152. # cf = torch.bincount(c.long(), minlength=nc) + 1.
  153. # model._initialize_biases(cf.to(device))
  154. if tb_writer:
  155. plot_labels(labels)
  156. tb_writer.add_histogram('classes', c, 0)
  157. # Check anchors
  158. if not opt.noautoanchor:
  159. check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz)
  160. # Exponential moving average
  161. ema = torch_utils.ModelEMA(model)
  162. # Start training
  163. t0 = time.time()
  164. nb = len(dataloader) # number of batches
  165. n_burn = max(3 * nb, 1e3) # burn-in iterations, max(3 epochs, 1k iterations)
  166. maps = np.zeros(nc) # mAP per class
  167. results = (0, 0, 0, 0, 0, 0, 0) # 'P', 'R', 'mAP', 'F1', 'val GIoU', 'val Objectness', 'val Classification'
  168. print('Image sizes %g train, %g test' % (imgsz, imgsz_test))
  169. print('Using %g dataloader workers' % dataloader.num_workers)
  170. print('Starting training for %g epochs...' % epochs)
  171. # torch.autograd.set_detect_anomaly(True)
  172. for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------
  173. model.train()
  174. # Update image weights (optional)
  175. if dataset.image_weights:
  176. w = model.class_weights.cpu().numpy() * (1 - maps) ** 2 # class weights
  177. image_weights = labels_to_image_weights(dataset.labels, nc=nc, class_weights=w)
  178. dataset.indices = random.choices(range(dataset.n), weights=image_weights, k=dataset.n) # rand weighted idx
  179. # Update mosaic border
  180. # b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs)
  181. # dataset.mosaic_border = [b - imgsz, -b] # height, width borders
  182. mloss = torch.zeros(4, device=device) # mean losses
  183. print(('\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'GIoU', 'obj', 'cls', 'total', 'targets', 'img_size'))
  184. pbar = tqdm(enumerate(dataloader), total=nb) # progress bar
  185. for i, (imgs, targets, paths, _) in pbar: # batch -------------------------------------------------------------
  186. ni = i + nb * epoch # number integrated batches (since train start)
  187. imgs = imgs.to(device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0
  188. # Burn-in
  189. if ni <= n_burn:
  190. xi = [0, n_burn] # x interp
  191. # model.gr = np.interp(ni, xi, [0.0, 1.0]) # giou loss ratio (obj_loss = 1.0 or giou)
  192. accumulate = max(1, np.interp(ni, xi, [1, nbs / batch_size]).round())
  193. for j, x in enumerate(optimizer.param_groups):
  194. # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
  195. x['lr'] = np.interp(ni, xi, [0.1 if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])
  196. if 'momentum' in x:
  197. x['momentum'] = np.interp(ni, xi, [0.9, hyp['momentum']])
  198. # Multi-scale
  199. if opt.multi_scale:
  200. sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size
  201. sf = sz / max(imgs.shape[2:]) # scale factor
  202. if sf != 1:
  203. ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple)
  204. imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False)
  205. # Forward
  206. pred = model(imgs)
  207. # Loss
  208. loss, loss_items = compute_loss(pred, targets.to(device), model)
  209. if not torch.isfinite(loss):
  210. print('WARNING: non-finite loss, ending training ', loss_items)
  211. return results
  212. # Backward
  213. if mixed_precision:
  214. with amp.scale_loss(loss, optimizer) as scaled_loss:
  215. scaled_loss.backward()
  216. else:
  217. loss.backward()
  218. # Optimize
  219. if ni % accumulate == 0:
  220. optimizer.step()
  221. optimizer.zero_grad()
  222. ema.update(model)
  223. # Print
  224. mloss = (mloss * i + loss_items) / (i + 1) # update mean losses
  225. mem = '%.3gG' % (torch.cuda.memory_cached() / 1E9 if torch.cuda.is_available() else 0) # (GB)
  226. s = ('%10s' * 2 + '%10.4g' * 6) % (
  227. '%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1])
  228. pbar.set_description(s)
  229. # Plot
  230. if ni < 3:
  231. f = 'train_batch%g.jpg' % ni # filename
  232. result = plot_images(images=imgs, targets=targets, paths=paths, fname=f)
  233. if tb_writer and result is not None:
  234. tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch)
  235. # tb_writer.add_graph(model, imgs) # add model to tensorboard
  236. # end batch ------------------------------------------------------------------------------------------------
  237. # Scheduler
  238. scheduler.step()
  239. # mAP
  240. ema.update_attr(model)
  241. final_epoch = epoch + 1 == epochs
  242. if not opt.notest or final_epoch: # Calculate mAP
  243. results, maps, times = test.test(opt.data,
  244. batch_size=batch_size,
  245. imgsz=imgsz_test,
  246. save_json=final_epoch and opt.data.endswith(os.sep + 'coco.yaml'),
  247. model=ema.ema,
  248. single_cls=opt.single_cls,
  249. dataloader=testloader)
  250. # Write
  251. with open(results_file, 'a') as f:
  252. f.write(s + '%10.4g' * 7 % results + '\n') # P, R, mAP, F1, test_losses=(GIoU, obj, cls)
  253. if len(opt.name) and opt.bucket:
  254. os.system('gsutil cp results.txt gs://%s/results/results%s.txt' % (opt.bucket, opt.name))
  255. # Tensorboard
  256. if tb_writer:
  257. tags = ['train/giou_loss', 'train/obj_loss', 'train/cls_loss',
  258. 'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/F1',
  259. 'val/giou_loss', 'val/obj_loss', 'val/cls_loss']
  260. for x, tag in zip(list(mloss[:-1]) + list(results), tags):
  261. tb_writer.add_scalar(tag, x, epoch)
  262. # Update best mAP
  263. fi = fitness(np.array(results).reshape(1, -1)) # fitness_i = weighted combination of [P, R, mAP, F1]
  264. if fi > best_fitness:
  265. best_fitness = fi
  266. # Save model
  267. save = (not opt.nosave) or (final_epoch and not opt.evolve)
  268. if save:
  269. with open(results_file, 'r') as f: # create checkpoint
  270. ckpt = {'epoch': epoch,
  271. 'best_fitness': best_fitness,
  272. 'training_results': f.read(),
  273. 'model': ema.ema.module if hasattr(model, 'module') else ema.ema,
  274. 'optimizer': None if final_epoch else optimizer.state_dict()}
  275. # Save last, best and delete
  276. torch.save(ckpt, last)
  277. if (best_fitness == fi) and not final_epoch:
  278. torch.save(ckpt, best)
  279. del ckpt
  280. # end epoch ----------------------------------------------------------------------------------------------------
  281. # end training
  282. n = opt.name
  283. if len(n):
  284. n = '_' + n if not n.isnumeric() else n
  285. fresults, flast, fbest = 'results%s.txt' % n, wdir + 'last%s.pt' % n, wdir + 'best%s.pt' % n
  286. for f1, f2 in zip([wdir + 'last.pt', wdir + 'best.pt', 'results.txt'], [flast, fbest, fresults]):
  287. if os.path.exists(f1):
  288. os.rename(f1, f2) # rename
  289. ispt = f2.endswith('.pt') # is *.pt
  290. strip_optimizer(f2) if ispt else None # strip optimizer
  291. os.system('gsutil cp %s gs://%s/weights' % (f2, opt.bucket)) if opt.bucket and ispt else None # upload
  292. if not opt.evolve:
  293. plot_results() # save as results.png
  294. print('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600))
  295. dist.destroy_process_group() if device.type != 'cpu' and torch.cuda.device_count() > 1 else None
  296. torch.cuda.empty_cache()
  297. return results
  298. if __name__ == '__main__':
  299. check_git_status()
  300. parser = argparse.ArgumentParser()
  301. parser.add_argument('--epochs', type=int, default=300)
  302. parser.add_argument('--batch-size', type=int, default=16)
  303. parser.add_argument('--cfg', type=str, default='models/yolov5s.yaml', help='*.cfg path')
  304. parser.add_argument('--data', type=str, default='data/coco128.yaml', help='*.data path')
  305. parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='train,test sizes')
  306. parser.add_argument('--rect', action='store_true', help='rectangular training')
  307. parser.add_argument('--resume', action='store_true', help='resume training from last.pt')
  308. parser.add_argument('--nosave', action='store_true', help='only save final checkpoint')
  309. parser.add_argument('--notest', action='store_true', help='only test final epoch')
  310. parser.add_argument('--noautoanchor', action='store_true', help='disable autoanchor check')
  311. parser.add_argument('--evolve', action='store_true', help='evolve hyperparameters')
  312. parser.add_argument('--bucket', type=str, default='', help='gsutil bucket')
  313. parser.add_argument('--cache-images', action='store_true', help='cache images for faster training')
  314. parser.add_argument('--weights', type=str, default='', help='initial weights path')
  315. parser.add_argument('--name', default='', help='renames results.txt to results_name.txt if supplied')
  316. parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
  317. parser.add_argument('--adam', action='store_true', help='use adam optimizer')
  318. parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%')
  319. parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
  320. opt = parser.parse_args()
  321. opt.weights = last if opt.resume else opt.weights
  322. opt.cfg = check_file(opt.cfg) # check file
  323. opt.data = check_file(opt.data) # check file
  324. print(opt)
  325. opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test)
  326. device = torch_utils.select_device(opt.device, apex=mixed_precision, batch_size=opt.batch_size)
  327. if device.type == 'cpu':
  328. mixed_precision = False
  329. # Train
  330. if not opt.evolve:
  331. tb_writer = SummaryWriter(comment=opt.name)
  332. print('Start Tensorboard with "tensorboard --logdir=runs", view at http://localhost:6006/')
  333. train(hyp)
  334. # Evolve hyperparameters (optional)
  335. else:
  336. tb_writer = None
  337. opt.notest, opt.nosave = True, True # only test/save final epoch
  338. if opt.bucket:
  339. os.system('gsutil cp gs://%s/evolve.txt .' % opt.bucket) # download evolve.txt if exists
  340. for _ in range(10): # generations to evolve
  341. if os.path.exists('evolve.txt'): # if evolve.txt exists: select best hyps and mutate
  342. # Select parent(s)
  343. parent = 'single' # parent selection method: 'single' or 'weighted'
  344. x = np.loadtxt('evolve.txt', ndmin=2)
  345. n = min(5, len(x)) # number of previous results to consider
  346. x = x[np.argsort(-fitness(x))][:n] # top n mutations
  347. w = fitness(x) - fitness(x).min() # weights
  348. if parent == 'single' or len(x) == 1:
  349. # x = x[random.randint(0, n - 1)] # random selection
  350. x = x[random.choices(range(n), weights=w)[0]] # weighted selection
  351. elif parent == 'weighted':
  352. x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination
  353. # Mutate
  354. mp, s = 0.9, 0.2 # mutation probability, sigma
  355. npr = np.random
  356. npr.seed(int(time.time()))
  357. g = np.array([1, 1, 1, 1, 1, 1, 1, 0, .1, 1, 0, 1, 1, 1, 1, 1, 1, 1]) # gains
  358. ng = len(g)
  359. v = np.ones(ng)
  360. while all(v == 1): # mutate until a change occurs (prevent duplicates)
  361. v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0)
  362. for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300)
  363. hyp[k] = x[i + 7] * v[i] # mutate
  364. # Clip to limits
  365. keys = ['lr0', 'iou_t', 'momentum', 'weight_decay', 'hsv_s', 'hsv_v', 'translate', 'scale', 'fl_gamma']
  366. limits = [(1e-5, 1e-2), (0.00, 0.70), (0.60, 0.98), (0, 0.001), (0, .9), (0, .9), (0, .9), (0, .9), (0, 3)]
  367. for k, v in zip(keys, limits):
  368. hyp[k] = np.clip(hyp[k], v[0], v[1])
  369. # Train mutation
  370. results = train(hyp.copy())
  371. # Write mutation results
  372. print_mutation(hyp, results, opt.bucket)
  373. # Plot results
  374. # plot_evolution_results(hyp)