From 47f6baf3e9899047f6dc7e0d7d50ccd2c1e3c998 Mon Sep 17 00:00:00 2001 From: Chiheb Ben Cheikh <100644963+ChihebBENCHEIKH1@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:31:25 +0100 Subject: [PATCH 1/2] [HfFileSystem] Replace cached listings instead of appending to them _ls_tree appended each fetched entry to its parent's dircache listing. Cached listings got duplicates, and refresh=True kept deleted files. A listing that failed halfway stayed cached as if it were complete. On buckets, ls() on a file cached that file alone as its folder's listing. Build the listings while iterating, then store them once the listing is complete. Skip the entry for the listed path itself, which only buckets return. --- src/huggingface_hub/hf_file_system.py | 6 ++- tests/test_hf_file_system.py | 68 ++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index b5558c3a3f..036c509e00 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -613,6 +613,7 @@ def _ls_tree( revision=resolved_path.revision, repo_type=resolved_path.repo_type, ) + listings: dict[str, list[dict[str, Any]]] = {} for path_info in tree: cache_path = root_path + "/" + path_info.path if isinstance(path_info, RepoFile): @@ -650,11 +651,12 @@ def _ls_tree( "type": "directory", "uploaded_at": path_info.uploaded_at, } - parent_path = self._parent(cache_path_info["name"]) - self.dircache.setdefault(parent_path, []).append(cache_path_info) + if cache_path != path: + listings.setdefault(self._parent(cache_path), []).append(cache_path_info) depth = cache_path[len(path) :].count("/") if maxdepth is None or depth <= maxdepth: out.append(cache_path_info) + self.dircache.update(listings) return out def _list_bucket_tree_with_folders( diff --git a/tests/test_hf_file_system.py b/tests/test_hf_file_system.py index 9084de5f70..50985c7de0 100644 --- a/tests/test_hf_file_system.py +++ b/tests/test_hf_file_system.py @@ -6,14 +6,14 @@ import os import pickle import tempfile -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Iterable, Optional, Type from unittest.mock import Mock, patch import fsspec import pytest -from huggingface_hub import HfApi, constants, hf_file_system +from huggingface_hub import BucketFile, HfApi, RepoFile, RepoFolder, constants, hf_file_system from huggingface_hub.errors import BucketNotFoundError, RepositoryNotFoundError, RevisionNotFoundError from huggingface_hub.hf_file_system import ( HfFileSystem, @@ -865,6 +865,29 @@ def _inner(bucket_id: str, *_, **kwargs): return patch.object(fs._api, "bucket_info", _inner) +def mock_list_repo_tree(fs: HfFileSystem, files: list[str]): + def _inner(repo_id: str, path_in_repo: str = "", *, recursive: bool = False, **kwargs): + folders = {str(parent) for file in files for parent in PurePosixPath(file).parents if parent.name} + prefix = f"{path_in_repo}/" if path_in_repo else "" + for path in sorted(folders | set(files)): + if path.startswith(prefix) and (recursive or "/" not in path.removeprefix(prefix)): + if path in folders: + yield RepoFolder(path=path, oid="0" * 40) + else: + yield RepoFile(path=path, size=1, oid="0" * 40) + + return patch.object(fs._api, "list_repo_tree", _inner) + + +def mock_list_bucket_tree(fs: HfFileSystem, files: list[str]): + def _inner(bucket_id: str, prefix: str = "", *, recursive: bool = False, **kwargs): + for file in files: + if file == prefix or file.startswith(f"{prefix}/"): + yield BucketFile(type="file", path=file, size=1, xetHash="0" * 64) + + return patch.object(fs._api, "list_bucket_tree", _inner) + + def test_resolve_path_with_non_matching_revisions(): fs = HfFileSystem() with pytest.raises(ValueError): @@ -900,6 +923,47 @@ def test_access_repositories_lists(not_supported_path, expected_error: Type[Exce fs.open(not_supported_path) +def test_ls_replaces_cached_listing(): + fs = HfFileSystem(skip_instance_cache=True) + files = ["README.md", "data/a.csv", "data/b.csv"] + with mock_repo_info(fs), mock_list_repo_tree(fs, files): + fs.ls("username/my_model/data") + fs.find("username/my_model") + assert fs.ls("username/my_model/data", detail=False) == [ + "username/my_model/data/a.csv", + "username/my_model/data/b.csv", + ] + + files.remove("data/b.csv") + fs.ls("username/my_model/data", refresh=True) + assert fs.ls("username/my_model/data", detail=False) == ["username/my_model/data/a.csv"] + + +def test_ls_does_not_cache_partial_listing(): + fs = HfFileSystem(skip_instance_cache=True) + + def _list_repo_tree(*args, **kwargs): + yield RepoFile(path="a.txt", size=1, oid="0" * 40) + raise ConnectionError + + with mock_repo_info(fs), patch.object(fs._api, "list_repo_tree", _list_repo_tree): + with pytest.raises(ConnectionError): + fs.ls("username/my_model") + assert fs.dircache == {} + + +def test_ls_bucket_file_does_not_hide_siblings(): + fs = HfFileSystem(skip_instance_cache=True) + with mock_bucket_info(fs), mock_list_bucket_tree(fs, ["data/a.csv", "data/b.csv"]): + assert fs.ls("buckets/username/my_bucket/data/a.csv", detail=False) == [ + "buckets/username/my_bucket/data/a.csv" + ] + assert fs.ls("buckets/username/my_bucket/data", detail=False) == [ + "buckets/username/my_bucket/data/a.csv", + "buckets/username/my_bucket/data/b.csv", + ] + + def test_exists_after_repo_deletion(repo_factory: RepoFactory): """Test that exists() correctly reflects repository deletion.""" # Initialize with staging endpoint and skip cache From 9e5b3d868234983d1ff7bfa3a2249b34dd305b6f Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Wed, 7 Oct 2026 11:17:21 +0200 Subject: [PATCH 2/2] fix tests: replace mocked listings with a staging test --- src/huggingface_hub/hf_file_system.py | 3 +- tests/test_hf_file_system.py | 76 ++++----------------------- 2 files changed, 12 insertions(+), 67 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 036c509e00..6be92fd89a 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -651,11 +651,12 @@ def _ls_tree( "type": "directory", "uploaded_at": path_info.uploaded_at, } - if cache_path != path: + if cache_path != path: # listing a bucket file returns the file, not its folder's listing listings.setdefault(self._parent(cache_path), []).append(cache_path_info) depth = cache_path[len(path) :].count("/") if maxdepth is None or depth <= maxdepth: out.append(cache_path_info) + # replace cached listings, only once the listing is complete self.dircache.update(listings) return out diff --git a/tests/test_hf_file_system.py b/tests/test_hf_file_system.py index 50985c7de0..56cc2b4cff 100644 --- a/tests/test_hf_file_system.py +++ b/tests/test_hf_file_system.py @@ -6,14 +6,14 @@ import os import pickle import tempfile -from pathlib import Path, PurePosixPath +from pathlib import Path from typing import Iterable, Optional, Type from unittest.mock import Mock, patch import fsspec import pytest -from huggingface_hub import BucketFile, HfApi, RepoFile, RepoFolder, constants, hf_file_system +from huggingface_hub import HfApi, constants, hf_file_system from huggingface_hub.errors import BucketNotFoundError, RepositoryNotFoundError, RevisionNotFoundError from huggingface_hub.hf_file_system import ( HfFileSystem, @@ -259,6 +259,14 @@ def test_list_data_file(self): self._check_info_not_expanded(files[0]) self._check_info_fields(files[0]) + def test_list_data_directory_after_listing_subpaths(self): + """Regression test for #5098.""" + data_dir = self.hf_path + "/data" + self.hffs.ls(self.text_file) + self.hffs.find(self.hf_path) + self.hffs.ls(data_dir, refresh=True) + assert sorted(self.hffs.ls(data_dir, detail=False)) == [data_dir + "/binary_data.bin", self.text_file] + def test_list_root_directory_no_detail_then_with_detail(self): files = sorted(self.hffs.ls(self.hf_path, detail=False)) assert len(files) == 2 @@ -865,29 +873,6 @@ def _inner(bucket_id: str, *_, **kwargs): return patch.object(fs._api, "bucket_info", _inner) -def mock_list_repo_tree(fs: HfFileSystem, files: list[str]): - def _inner(repo_id: str, path_in_repo: str = "", *, recursive: bool = False, **kwargs): - folders = {str(parent) for file in files for parent in PurePosixPath(file).parents if parent.name} - prefix = f"{path_in_repo}/" if path_in_repo else "" - for path in sorted(folders | set(files)): - if path.startswith(prefix) and (recursive or "/" not in path.removeprefix(prefix)): - if path in folders: - yield RepoFolder(path=path, oid="0" * 40) - else: - yield RepoFile(path=path, size=1, oid="0" * 40) - - return patch.object(fs._api, "list_repo_tree", _inner) - - -def mock_list_bucket_tree(fs: HfFileSystem, files: list[str]): - def _inner(bucket_id: str, prefix: str = "", *, recursive: bool = False, **kwargs): - for file in files: - if file == prefix or file.startswith(f"{prefix}/"): - yield BucketFile(type="file", path=file, size=1, xetHash="0" * 64) - - return patch.object(fs._api, "list_bucket_tree", _inner) - - def test_resolve_path_with_non_matching_revisions(): fs = HfFileSystem() with pytest.raises(ValueError): @@ -923,47 +908,6 @@ def test_access_repositories_lists(not_supported_path, expected_error: Type[Exce fs.open(not_supported_path) -def test_ls_replaces_cached_listing(): - fs = HfFileSystem(skip_instance_cache=True) - files = ["README.md", "data/a.csv", "data/b.csv"] - with mock_repo_info(fs), mock_list_repo_tree(fs, files): - fs.ls("username/my_model/data") - fs.find("username/my_model") - assert fs.ls("username/my_model/data", detail=False) == [ - "username/my_model/data/a.csv", - "username/my_model/data/b.csv", - ] - - files.remove("data/b.csv") - fs.ls("username/my_model/data", refresh=True) - assert fs.ls("username/my_model/data", detail=False) == ["username/my_model/data/a.csv"] - - -def test_ls_does_not_cache_partial_listing(): - fs = HfFileSystem(skip_instance_cache=True) - - def _list_repo_tree(*args, **kwargs): - yield RepoFile(path="a.txt", size=1, oid="0" * 40) - raise ConnectionError - - with mock_repo_info(fs), patch.object(fs._api, "list_repo_tree", _list_repo_tree): - with pytest.raises(ConnectionError): - fs.ls("username/my_model") - assert fs.dircache == {} - - -def test_ls_bucket_file_does_not_hide_siblings(): - fs = HfFileSystem(skip_instance_cache=True) - with mock_bucket_info(fs), mock_list_bucket_tree(fs, ["data/a.csv", "data/b.csv"]): - assert fs.ls("buckets/username/my_bucket/data/a.csv", detail=False) == [ - "buckets/username/my_bucket/data/a.csv" - ] - assert fs.ls("buckets/username/my_bucket/data", detail=False) == [ - "buckets/username/my_bucket/data/a.csv", - "buckets/username/my_bucket/data/b.csv", - ] - - def test_exists_after_repo_deletion(repo_factory: RepoFactory): """Test that exists() correctly reflects repository deletion.""" # Initialize with staging endpoint and skip cache