zjc
/
yolov5
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add `retry=3` to `download()` (#7313) 

* Add `retry=3` to `download()`

* Update general.py

* Update general.py

* Update general.py

* Update VOC.yaml

* Update VisDrone.yaml
This commit is contained in:
Glenn Jocher 2022-04-06 13:12:41 +02:00 committed by GitHub
parent f735458987
commit 32661f75ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/VOC.yaml
View File

@ -62,7 +62,7 @@ download: |
urls = [url + 'VOCtrainval_06-Nov-2007.zip', # 446MB, 5012 images urls = [url + 'VOCtrainval_06-Nov-2007.zip', # 446MB, 5012 images
url + 'VOCtest_06-Nov-2007.zip', # 438MB, 4953 images url + 'VOCtest_06-Nov-2007.zip', # 438MB, 4953 images
url + 'VOCtrainval_11-May-2012.zip'] # 1.95GB, 17126 images url + 'VOCtrainval_11-May-2012.zip'] # 1.95GB, 17126 images
download(urls, dir=dir / 'images', delete=False, threads=3) download(urls, dir=dir / 'images', delete=False, curl=True, threads=3)
# Convert # Convert
path = dir / f'images/VOCdevkit' path = dir / f'images/VOCdevkit'

2
data/VisDrone.yaml
View File

@ -54,7 +54,7 @@ download: |
'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-val.zip', 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-val.zip',
'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-dev.zip', 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-dev.zip',
'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-challenge.zip'] 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-challenge.zip']
download(urls, dir=dir, threads=4) download(urls, dir=dir, curl=True, threads=4)
# Convert # Convert
for d in 'VisDrone2019-DET-train', 'VisDrone2019-DET-val', 'VisDrone2019-DET-test-dev': for d in 'VisDrone2019-DET-train', 'VisDrone2019-DET-val', 'VisDrone2019-DET-test-dev':

18
utils/general.py
View File

@ -497,20 +497,32 @@ def url2file(url):
return file return file
def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1): def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1, retry=3):
# Multi-threaded file download and unzip function, used in data.yaml for autodownload # Multi-threaded file download and unzip function, used in data.yaml for autodownload
def download_one(url, dir): def download_one(url, dir):
# Download 1 file # Download 1 file
success = True
f = dir / Path(url).name # filename f = dir / Path(url).name # filename
if Path(url).is_file(): # exists in current path if Path(url).is_file(): # exists in current path
Path(url).rename(f) # move to dir Path(url).rename(f) # move to dir
elif not f.exists(): elif not f.exists():
LOGGER.info(f'Downloading {url} to {f}...') LOGGER.info(f'Downloading {url} to {f}...')
for i in range(retry + 1):
if curl: if curl:
os.system(f"curl -L '{url}' -o '{f}' --retry 9 -C -") # curl download, retry and resume on fail s = 'sS' if threads > 1 else '' # silent
r = os.system(f"curl -{s}L '{url}' -o '{f}' --retry 9 -C -") # curl download
success = r == 0
else: else:
torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download
if unzip and f.suffix in ('.zip', '.gz'): success = f.is_file()
if success:
break
elif i < retry:
LOGGER.warning(f'Download failure, retrying {i + 1}/{retry} {url}...')
else:
LOGGER.warning(f'Failed to download {url}...')
if unzip and success and f.suffix in ('.zip', '.gz'):
LOGGER.info(f'Unzipping {f}...') LOGGER.info(f'Unzipping {f}...')
if f.suffix == '.zip': if f.suffix == '.zip':
ZipFile(f).extractall(path=dir) # unzip ZipFile(f).extractall(path=dir) # unzip