Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[HfFileSystem] Replace cached listings instead of appending to them
_ls_tree appended each fetched entry to its parent's dircache listing.
Cached listings got duplicates, and refresh=True kept deleted files.
A listing that failed halfway stayed cached as if it were complete.
On buckets, ls() on a file cached that file alone as its folder's listing.

Build the listings while iterating, then store them once the listing is
complete. Skip the entry for the listed path itself, which only buckets
return.
  • Loading branch information
ChihebBENCHEIKH1 committed Oct 6, 2026
commit 47f6baf3e9899047f6dc7e0d7d50ccd2c1e3c998
6 changes: 4 additions & 2 deletions src/huggingface_hub/hf_file_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,7 @@ def _ls_tree(
revision=resolved_path.revision,
repo_type=resolved_path.repo_type,
)
listings: dict[str, list[dict[str, Any]]] = {}
for path_info in tree:
cache_path = root_path + "/" + path_info.path
if isinstance(path_info, RepoFile):
Expand Down Expand Up @@ -650,11 +651,12 @@ def _ls_tree(
"type": "directory",
"uploaded_at": path_info.uploaded_at,
}
parent_path = self._parent(cache_path_info["name"])
self.dircache.setdefault(parent_path, []).append(cache_path_info)
if cache_path != path:
listings.setdefault(self._parent(cache_path), []).append(cache_path_info)
depth = cache_path[len(path) :].count("/")
if maxdepth is None or depth <= maxdepth:
out.append(cache_path_info)
self.dircache.update(listings)
return out

def _list_bucket_tree_with_folders(
Expand Down
68 changes: 66 additions & 2 deletions tests/test_hf_file_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@
import os
import pickle
import tempfile
from pathlib import Path
from pathlib import Path, PurePosixPath
from typing import Iterable, Optional, Type
from unittest.mock import Mock, patch

import fsspec
import pytest

from huggingface_hub import HfApi, constants, hf_file_system
from huggingface_hub import BucketFile, HfApi, RepoFile, RepoFolder, constants, hf_file_system
from huggingface_hub.errors import BucketNotFoundError, RepositoryNotFoundError, RevisionNotFoundError
from huggingface_hub.hf_file_system import (
HfFileSystem,
Expand Down Expand Up @@ -865,6 +865,29 @@ def _inner(bucket_id: str, *_, **kwargs):
return patch.object(fs._api, "bucket_info", _inner)


def mock_list_repo_tree(fs: HfFileSystem, files: list[str]):
def _inner(repo_id: str, path_in_repo: str = "", *, recursive: bool = False, **kwargs):
folders = {str(parent) for file in files for parent in PurePosixPath(file).parents if parent.name}
prefix = f"{path_in_repo}/" if path_in_repo else ""
for path in sorted(folders | set(files)):
if path.startswith(prefix) and (recursive or "/" not in path.removeprefix(prefix)):
if path in folders:
yield RepoFolder(path=path, oid="0" * 40)
else:
yield RepoFile(path=path, size=1, oid="0" * 40)

return patch.object(fs._api, "list_repo_tree", _inner)


def mock_list_bucket_tree(fs: HfFileSystem, files: list[str]):
def _inner(bucket_id: str, prefix: str = "", *, recursive: bool = False, **kwargs):
for file in files:
if file == prefix or file.startswith(f"{prefix}/"):
yield BucketFile(type="file", path=file, size=1, xetHash="0" * 64)

return patch.object(fs._api, "list_bucket_tree", _inner)


def test_resolve_path_with_non_matching_revisions():
fs = HfFileSystem()
with pytest.raises(ValueError):
Expand Down Expand Up @@ -900,6 +923,47 @@ def test_access_repositories_lists(not_supported_path, expected_error: Type[Exce
fs.open(not_supported_path)


def test_ls_replaces_cached_listing():
fs = HfFileSystem(skip_instance_cache=True)
files = ["README.md", "data/a.csv", "data/b.csv"]
with mock_repo_info(fs), mock_list_repo_tree(fs, files):
fs.ls("username/my_model/data")
fs.find("username/my_model")
assert fs.ls("username/my_model/data", detail=False) == [
"username/my_model/data/a.csv",
"username/my_model/data/b.csv",
]

files.remove("data/b.csv")
fs.ls("username/my_model/data", refresh=True)
assert fs.ls("username/my_model/data", detail=False) == ["username/my_model/data/a.csv"]


def test_ls_does_not_cache_partial_listing():
fs = HfFileSystem(skip_instance_cache=True)

def _list_repo_tree(*args, **kwargs):
yield RepoFile(path="a.txt", size=1, oid="0" * 40)
raise ConnectionError

with mock_repo_info(fs), patch.object(fs._api, "list_repo_tree", _list_repo_tree):
with pytest.raises(ConnectionError):
fs.ls("username/my_model")
assert fs.dircache == {}


def test_ls_bucket_file_does_not_hide_siblings():
fs = HfFileSystem(skip_instance_cache=True)
with mock_bucket_info(fs), mock_list_bucket_tree(fs, ["data/a.csv", "data/b.csv"]):
assert fs.ls("buckets/username/my_bucket/data/a.csv", detail=False) == [
"buckets/username/my_bucket/data/a.csv"
]
assert fs.ls("buckets/username/my_bucket/data", detail=False) == [
"buckets/username/my_bucket/data/a.csv",
"buckets/username/my_bucket/data/b.csv",
]


def test_exists_after_repo_deletion(repo_factory: RepoFactory):
"""Test that exists() correctly reflects repository deletion."""
# Initialize with staging endpoint and skip cache
Expand Down