W&B: Fix dataset check (#4879)
* evolve fix * Enable login timeout * fix pkg * check rank * don't relogin * fix * reformat
This commit is contained in:
parent
9febea79de
commit
59aae85a7e
|
|
@ -5,8 +5,8 @@ import os
|
||||||
import sys
|
import sys
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import pkg_resources as pkg
|
|
||||||
|
|
||||||
|
import pkg_resources as pkg
|
||||||
import yaml
|
import yaml
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
|
||||||
|
|
@ -49,9 +49,11 @@ def check_wandb_dataset(data_file):
|
||||||
if check_file(data_file) and data_file.endswith('.yaml'):
|
if check_file(data_file) and data_file.endswith('.yaml'):
|
||||||
with open(data_file, errors='ignore') as f:
|
with open(data_file, errors='ignore') as f:
|
||||||
data_dict = yaml.safe_load(f)
|
data_dict = yaml.safe_load(f)
|
||||||
is_wandb_artifact = (data_dict['train'].startswith(WANDB_ARTIFACT_PREFIX) or
|
is_trainset_wandb_artifact = (isinstance(data_dict['train'], str) and
|
||||||
|
data_dict['train'].startswith(WANDB_ARTIFACT_PREFIX))
|
||||||
|
is_valset_wandb_artifact = (isinstance(data_dict['val'], str) and
|
||||||
data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX))
|
data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX))
|
||||||
if is_wandb_artifact:
|
if is_trainset_wandb_artifact or is_valset_wandb_artifact:
|
||||||
return data_dict
|
return data_dict
|
||||||
else:
|
else:
|
||||||
return check_dataset(data_file)
|
return check_dataset(data_file)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue