Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions espnet_model_zoo/downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,18 @@ def _resolve_huggingface(name, url):
test here, download() then recognised it and returned a snapshot
directory, and download_and_unpack() tried to unpack that directory as an
archive. All three now come out as (repo_id, "https://huggingface.co/").

A row whose url is the marker itself, "https://huggingface.co/", is the
first shape, not the third: the URL names no repository, so the id stays
the name. Taking the empty tail as the id sent "" to snapshot_download.
"""
for candidate in (name, url):
if isinstance(candidate, str) and candidate.startswith(
"https://huggingface.co/"
):
repo_id = candidate[len("https://huggingface.co/") :]
return repo_id, "https://huggingface.co/"
repo_id = candidate[len("https://huggingface.co/") :].strip("/")
if repo_id:
return repo_id, "https://huggingface.co/"
return name, url


Expand Down
24 changes: 24 additions & 0 deletions test/test_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,30 @@ def fake_huggingface_download(name=None, **kw):
assert d.download_and_unpack(marker_tag) == {"cache_dir": marker_tag}


def test_table_row_with_bare_hub_marker_url_keeps_its_name(tmp_path, monkeypatch):
# Four table.csv rows hold "https://huggingface.co/" - the marker with a
# trailing slash - as their url. Read as a full repository URL, its empty
# tail became the repo id and snapshot_download rejected "".
tag = "reazon-research/reazonspeech-espnet-v1"
d = ModelDownloader(tmp_path)
assert d.get_url(tag) == "https://huggingface.co/"
seen = []

def fake_huggingface_download(name=None, **kw):
seen.append(name)
return name

monkeypatch.setattr(d, "huggingface_download", fake_huggingface_download)
monkeypatch.setattr(
d,
"_unpack_cache_dir_for_huggingface",
lambda cache_dir: {"cache_dir": cache_dir},
)
assert d.download_and_unpack(tag) == {"cache_dir": tag}
assert d.download(tag) == tag
assert seen == [tag, tag]


def _fake_snapshot(root: Path):
# The shape of a packed espnet model as huggingface_hub lays it out: a
# meta.yaml naming the config and weights, a config whose paths are
Expand Down
Loading