|
|
@@ -1,6 +1,7 @@ |
|
|
|
# Dataset utils and dataloaders |
|
|
|
|
|
|
|
import glob |
|
|
|
import hashlib |
|
|
|
import logging |
|
|
|
import math |
|
|
|
import os |
|
|
@@ -36,9 +37,12 @@ for orientation in ExifTags.TAGS.keys(): |
|
|
|
break |
|
|
|
|
|
|
|
|
|
|
|
def get_hash(files): |
|
|
|
# Returns a single hash value of a list of files |
|
|
|
return sum(os.path.getsize(f) for f in files if os.path.isfile(f)) |
|
|
|
def get_hash(paths): |
|
|
|
# Returns a single hash value of a list of paths (files or dirs) |
|
|
|
size = sum(os.path.getsize(p) for p in paths if os.path.exists(p)) # sizes |
|
|
|
h = hashlib.md5(str(size).encode()) # hash sizes |
|
|
|
h.update(''.join(paths).encode()) # hash paths |
|
|
|
return h.hexdigest() # return hash |
|
|
|
|
|
|
|
|
|
|
|
def exif_size(img): |
|
|
@@ -383,7 +387,7 @@ class LoadImagesAndLabels(Dataset): # for training/testing |
|
|
|
cache_path = (p if p.is_file() else Path(self.label_files[0]).parent).with_suffix('.cache') # cached labels |
|
|
|
if cache_path.is_file(): |
|
|
|
cache, exists = torch.load(cache_path), True # load |
|
|
|
if cache['hash'] != get_hash(self.label_files + self.img_files) or 'version' not in cache: # changed |
|
|
|
if cache['hash'] != get_hash(self.label_files + self.img_files): # changed |
|
|
|
cache, exists = self.cache_labels(cache_path, prefix), False # re-cache |
|
|
|
else: |
|
|
|
cache, exists = self.cache_labels(cache_path, prefix), False # cache |
|
|
@@ -501,9 +505,9 @@ class LoadImagesAndLabels(Dataset): # for training/testing |
|
|
|
|
|
|
|
x['hash'] = get_hash(self.label_files + self.img_files) |
|
|
|
x['results'] = nf, nm, ne, nc, i + 1 |
|
|
|
x['version'] = 0.1 # cache version |
|
|
|
x['version'] = 0.2 # cache version |
|
|
|
try: |
|
|
|
torch.save(x, path) # save for next time |
|
|
|
torch.save(x, path) # save cache for next time |
|
|
|
logging.info(f'{prefix}New cache created: {path}') |
|
|
|
except Exception as e: |
|
|
|
logging.info(f'{prefix}WARNING: Cache directory {path.parent} is not writeable: {e}') # path not writeable |