fix: recognize root Hugging Face repo IDs (#325)
* fix: recognize root Hugging Face repo IDs * fix: propagate invalid HF repo ids * fix: match transformers local path precedence
This commit is contained in:
@@ -22,6 +22,7 @@ from datasets import DatasetDict, ReadInstruction, load_dataset, load_from_disk
|
|||||||
from datasets.config import DATASET_STATE_JSON_FILENAME
|
from datasets.config import DATASET_STATE_JSON_FILENAME
|
||||||
from datasets.download.download_manager import DownloadMode
|
from datasets.download.download_manager import DownloadMode
|
||||||
from datasets.utils.info_utils import VerificationMode
|
from datasets.utils.info_utils import VerificationMode
|
||||||
|
from huggingface_hub.utils import validate_repo_id
|
||||||
from optuna import Trial
|
from optuna import Trial
|
||||||
from psutil import Process
|
from psutil import Process
|
||||||
from questionary import Choice, Style
|
from questionary import Choice, Style
|
||||||
@@ -172,13 +173,13 @@ def format_duration(seconds: float) -> str:
|
|||||||
def is_hf_path(path: str) -> bool:
|
def is_hf_path(path: str) -> bool:
|
||||||
"""Checks whether a path likely refers to a Hugging Face repository."""
|
"""Checks whether a path likely refers to a Hugging Face repository."""
|
||||||
|
|
||||||
return (
|
# Match Transformers: existing local paths take precedence over Hub lookup,
|
||||||
not path.startswith("/")
|
# even if the path string is also a valid repository ID.
|
||||||
and not path.endswith("/")
|
if Path(path).exists():
|
||||||
and path.count("/") == 1
|
return False
|
||||||
and "\\" not in path
|
|
||||||
and not Path(path).exists()
|
validate_repo_id(path)
|
||||||
)
|
return True
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
Reference in New Issue
Block a user