Browse Source

W&B DDP fix (#2574)

5.0
Ayush Chaurasia GitHub 3 years ago
parent
commit
1bf9365280
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 9 additions and 4 deletions
  1. +5
    -3
      train.py
  2. +4
    -1
      utils/wandb_logging/wandb_utils.py

+ 5
- 3
train.py View File

is_coco = opt.data.endswith('coco.yaml') is_coco = opt.data.endswith('coco.yaml')


# Logging- Doing this before checking the dataset. Might update data_dict # Logging- Doing this before checking the dataset. Might update data_dict
loggers = {'wandb': None} # loggers dict
if rank in [-1, 0]: if rank in [-1, 0]:
opt.hyp = hyp # add hyperparameters opt.hyp = hyp # add hyperparameters
run_id = torch.load(weights).get('wandb_id') if weights.endswith('.pt') and os.path.isfile(weights) else None run_id = torch.load(weights).get('wandb_id') if weights.endswith('.pt') and os.path.isfile(weights) else None
wandb_logger = WandbLogger(opt, Path(opt.save_dir).stem, run_id, data_dict) wandb_logger = WandbLogger(opt, Path(opt.save_dir).stem, run_id, data_dict)
loggers['wandb'] = wandb_logger.wandb
data_dict = wandb_logger.data_dict data_dict = wandb_logger.data_dict
if wandb_logger.wandb: if wandb_logger.wandb:
weights, epochs, hyp = opt.weights, opt.epochs, opt.hyp # WandbLogger might update weights, epochs if resuming weights, epochs, hyp = opt.weights, opt.epochs, opt.hyp # WandbLogger might update weights, epochs if resuming
loggers = {'wandb': wandb_logger.wandb} # loggers dict
nc = 1 if opt.single_cls else int(data_dict['nc']) # number of classes nc = 1 if opt.single_cls else int(data_dict['nc']) # number of classes
names = ['item'] if opt.single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names names = ['item'] if opt.single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names
assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check
fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95]
if fi > best_fitness: if fi > best_fitness:
best_fitness = fi best_fitness = fi
wandb_logger.end_epoch(best_result=best_fitness == fi)


# Save model # Save model
if (not opt.nosave) or (final_epoch and not opt.evolve): # if save if (not opt.nosave) or (final_epoch and not opt.evolve): # if save
wandb_logger.log_model( wandb_logger.log_model(
last.parent, opt, epoch, fi, best_model=best_fitness == fi) last.parent, opt, epoch, fi, best_model=best_fitness == fi)
del ckpt del ckpt
wandb_logger.end_epoch(best_result=best_fitness == fi)
# end epoch ---------------------------------------------------------------------------------------------------- # end epoch ----------------------------------------------------------------------------------------------------
# end training # end training
wandb_logger.wandb.log_artifact(str(final), type='model', wandb_logger.wandb.log_artifact(str(final), type='model',
name='run_' + wandb_logger.wandb_run.id + '_model', name='run_' + wandb_logger.wandb_run.id + '_model',
aliases=['last', 'best', 'stripped']) aliases=['last', 'best', 'stripped'])
wandb_logger.finish_run()
else: else:
dist.destroy_process_group() dist.destroy_process_group()
torch.cuda.empty_cache() torch.cuda.empty_cache()
wandb_logger.finish_run()
return results return results





+ 4
- 1
utils/wandb_logging/wandb_utils.py View File



try: try:
import wandb import wandb
from wandb import init, finish
except ImportError: except ImportError:
wandb = None wandb = None
print(f"{colorstr('wandb: ')}Install Weights & Biases for YOLOv5 logging with 'pip install wandb' (recommended)")


WANDB_ARTIFACT_PREFIX = 'wandb-artifact://' WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'


self.data_dict = self.setup_training(opt, data_dict) self.data_dict = self.setup_training(opt, data_dict)
if self.job_type == 'Dataset Creation': if self.job_type == 'Dataset Creation':
self.data_dict = self.check_and_upload_dataset(opt) self.data_dict = self.check_and_upload_dataset(opt)
else:
print(f"{colorstr('wandb: ')}Install Weights & Biases for YOLOv5 logging with 'pip install wandb' (recommended)")



def check_and_upload_dataset(self, opt): def check_and_upload_dataset(self, opt):
assert wandb, 'Install wandb to upload dataset' assert wandb, 'Install wandb to upload dataset'

Loading…
Cancel
Save