From 7a5328de3ee3e82a9e205c14c7db93765046c311 Mon Sep 17 00:00:00 2001 From: askalf <263217947+askalf@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:43:46 -0400 Subject: [PATCH 1/2] [Download] Support Windows long paths for the tree cache file `_tree_cache_path()` joined `/trees/.json` with no Windows extended-length handling, unlike every sibling path in `_local_folder.py`, `_snapshot_download.py` and `file_download.py`. On a deep `local_dir` or `cache_dir` (path over 255 characters with the default LongPathsEnabled=0) the write failed with WinError 206, `write_tree_cache` swallowed it, and the next read missed, so the tree cache was silently disabled and every download re-fetched the tree listing. Apply the extended-length prefix in the one function both the reader and the writer go through, so they agree on the on-disk path and the in-memory cache key. Drive paths become `\\?\C:\...`; UNC paths become `\\?\UNC\server\share\...`, because prefixing a UNC path verbatim produces `\\?\\\server\...`, which Windows rejects. Add two Windows-only tests: a disk round-trip parametrized at 260 (only the JSON file exceeds the limit) and 300 (the `trees/` folder does too), and a pure-function check of the drive, UNC, already-prefixed and short-path spellings. --- src/huggingface_hub/_tree_cache.py | 19 +++++++++++- tests/test_tree_cache.py | 50 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/huggingface_hub/_tree_cache.py b/src/huggingface_hub/_tree_cache.py index 7026bc41939..f2d411e01a4 100644 --- a/src/huggingface_hub/_tree_cache.py +++ b/src/huggingface_hub/_tree_cache.py @@ -85,8 +85,25 @@ def is_valid_tree_entries(entries: dict[str, TreeCacheEntry]) -> bool: return all(entry.xet_hash is None or is_valid_xet_hash(entry.xet_hash) for entry in entries.values()) +def _extended_length_path(absolute_path: str) -> str: + r"""Return a Windows absolute path in its extended-length form. + + A drive path becomes `\\?\C:\...`; a UNC path `\\server\share\...` becomes `\\?\UNC\server\share\...` + (prefixing a UNC path verbatim would produce an invalid `\\?\\\server` path). + """ + if absolute_path.startswith("\\\\"): + return "\\\\?\\UNC\\" + absolute_path[2:] + return "\\\\?\\" + absolute_path + + def _tree_cache_path(tree_cache_folder: str, commit_hash: str) -> str: - return os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json") + path = os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json") + # Some Windows versions do not allow for paths longer than 255 characters. + # In this case, we must specify it as an extended path by using the "\\?\" prefix. + # Both readers and writers go through here, so they agree on the path (and on the in-memory cache key). + if os.name == "nt" and len(os.path.abspath(path)) > 255 and not path.startswith("\\\\?\\"): + path = _extended_length_path(os.path.abspath(path)) + return path def tree_cache_folder_for_local_dir(local_dir: str) -> str: diff --git a/tests/test_tree_cache.py b/tests/test_tree_cache.py index c6516b94b1e..2d63ee8e540 100644 --- a/tests/test_tree_cache.py +++ b/tests/test_tree_cache.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +import os from pathlib import Path from unittest.mock import patch @@ -22,6 +23,7 @@ _IN_MEMORY_TREE_CACHE, TREE_CACHE_FORMAT_VERSION, TreeCacheEntry, + _tree_cache_path, read_tree_cache, tree_cache_folder_for_local_dir, write_tree_cache, @@ -121,6 +123,54 @@ def test_in_memory_cache_memoizes_first_read(self, tmp_path: Path): assert read_tree_cache(str(tmp_path), COMMIT_HASH) == _entries() mock_read.assert_not_called() + @pytest.mark.skipif(os.name != "nt", reason="Windows-specific test.") + @pytest.mark.parametrize("tree_cache_path_len", [260, 300]) + def test_round_trip_in_deep_folder(self, tmp_path: Path, tree_cache_path_len: int): + r"""A deep download target must still get a usable tree cache on Windows. + + Without long path support enabled, Windows caps file paths at 255 characters (and directory + paths at 247), so for a deep ``local_dir`` (or ``cache_dir``) writing + ``/trees/.json`` failed. `write_tree_cache` swallows that failure and the + next read simply misses, so the cache was silently disabled and every download re-fetched the + tree listing. Both boundary cases are covered: 260 (only the JSON file exceeds the limit, the + ``trees/`` folder still fits) and 300 (the ``trees/`` folder exceeds it too, so the `makedirs` + needs the prefix as well). + """ + # Pad the folder so that `/trees/.json` is `tree_cache_path_len` chars long. + # Use the extended-length prefix here since a plain mkdir of such a path would itself fail. + relative_depth = len(str(Path(".cache") / "huggingface" / "trees" / f"{COMMIT_HASH}.json")) + local_dir = tmp_path / ("d" * max(1, tree_cache_path_len - len(str(tmp_path)) - relative_depth - 2)) + os.makedirs("\\\\?\\" + os.path.abspath(local_dir), exist_ok=True) + folder = tree_cache_folder_for_local_dir(str(local_dir)) + assert len(str(Path(folder) / "trees" / f"{COMMIT_HASH}.json")) >= tree_cache_path_len + + write_tree_cache(folder, COMMIT_HASH, _entries()) + + # Drop the in-memory entry seeded by the write, so the read has to go through the disk. + _IN_MEMORY_TREE_CACHE.pop(_tree_cache_path(folder, COMMIT_HASH), None) + assert read_tree_cache(folder, COMMIT_HASH) == _entries() + + @pytest.mark.skipif(os.name != "nt", reason="Windows-specific test.") + def test_extended_length_prefix_for_drive_and_unc_paths(self): + r"""Long drive paths get the `\\?\` prefix, long UNC paths the `\\?\UNC\` form, short paths are untouched.""" + long_drive_folder = "C:\\" + "d" * 300 + assert _tree_cache_path(long_drive_folder, COMMIT_HASH) == ( + "\\\\?\\" + long_drive_folder + "\\trees\\" + f"{COMMIT_HASH}.json" + ) + + # A UNC share must not become `\\?\\\server\share` (invalid); it must become `\\?\UNC\server\share`. + long_unc_folder = "\\\\server\\share\\" + "d" * 300 + assert _tree_cache_path(long_unc_folder, COMMIT_HASH) == ( + "\\\\?\\UNC\\server\\share\\" + "d" * 300 + "\\trees\\" + f"{COMMIT_HASH}.json" + ) + + # Already-prefixed and short paths come back as-is: no double prefix, no behavior change. + already_prefixed = "\\\\?\\C:\\" + "d" * 300 + assert _tree_cache_path(already_prefixed, COMMIT_HASH).count("\\\\?\\") == 1 + assert _tree_cache_path("C:\\models\\mymodel", COMMIT_HASH) == os.path.join( + "C:\\models\\mymodel", "trees", f"{COMMIT_HASH}.json" + ) + @pytest.fixture def tree_cache_folder(tmp_path: Path): From 8f3ab5a42edba2feb98e0dc5d72e158c1b56caec Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Mon, 14 Sep 2026 12:38:10 +0200 Subject: [PATCH 2/2] [Download] Share a single extended-length path helper The `\\?\` prefixing was inlined 6 times across the download stack (`_local_folder.py`, `file_download.py`, `_snapshot_download.py`), so the tree cache fix added a 7th copy. Move it to `utils/_paths.as_extended_path()` and reuse it everywhere, UNC handling included. Co-Authored-By: Claude Opus 5 (1M context) --- src/huggingface_hub/_local_folder.py | 45 +++++++---------------- src/huggingface_hub/_snapshot_download.py | 6 +-- src/huggingface_hub/_tree_cache.py | 22 ++--------- src/huggingface_hub/file_download.py | 24 ++---------- src/huggingface_hub/utils/_paths.py | 28 ++++++++++++++ tests/test_tree_cache.py | 41 ++------------------- 6 files changed, 56 insertions(+), 110 deletions(-) diff --git a/src/huggingface_hub/_local_folder.py b/src/huggingface_hub/_local_folder.py index 8af934ca56e..4863292c574 100644 --- a/src/huggingface_hub/_local_folder.py +++ b/src/huggingface_hub/_local_folder.py @@ -57,6 +57,7 @@ from pathlib import Path, PurePosixPath, PureWindowsPath from .utils import WeakFileLock +from .utils._paths import as_extended_path logger = logging.getLogger(__name__) @@ -92,12 +93,7 @@ class LocalDownloadFilePaths: def incomplete_path(self, etag: str) -> Path: """Return the path where a file will be temporarily downloaded before being moved to `file_path`.""" path = self.metadata_path.parent / f"{_short_hash(self.metadata_path.name)}.{etag}.incomplete" - resolved_path = str(path.resolve()) - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if os.name == "nt" and len(resolved_path) > 255 and not resolved_path.startswith("\\\\?\\"): - path = Path("\\\\?\\" + resolved_path) - return path + return Path(as_extended_path(path)) @dataclass(frozen=True) @@ -253,13 +249,10 @@ def get_local_download_paths(local_dir: Path, filename: str) -> LocalDownloadFil metadata_path = _huggingface_dir(local_dir) / "download" / f"{sanitized_filename}.metadata" lock_path = metadata_path.with_suffix(".lock") - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix - if os.name == "nt": - if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: - file_path = Path("\\\\?\\" + os.path.abspath(file_path)) - lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) - metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) + # Parent directories are created below => use the stricter Windows limit for directories. + file_path = Path(as_extended_path(file_path, max_length=247)) + lock_path = Path(as_extended_path(lock_path, max_length=247)) + metadata_path = Path(as_extended_path(metadata_path, max_length=247)) file_path.parent.mkdir(parents=True, exist_ok=True) metadata_path.parent.mkdir(parents=True, exist_ok=True) @@ -288,13 +281,10 @@ def get_local_upload_paths(local_dir: Path, filename: str) -> LocalUploadFilePat metadata_path = _huggingface_dir(local_dir) / "upload" / f"{sanitized_filename}.metadata" lock_path = metadata_path.with_suffix(".lock") - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix - if os.name == "nt": - if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: - file_path = Path("\\\\?\\" + os.path.abspath(file_path)) - lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) - metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) + # Parent directories are created below => use the stricter Windows limit for directories. + file_path = Path(as_extended_path(file_path, max_length=247)) + lock_path = Path(as_extended_path(lock_path, max_length=247)) + metadata_path = Path(as_extended_path(metadata_path, max_length=247)) file_path.parent.mkdir(parents=True, exist_ok=True) metadata_path.parent.mkdir(parents=True, exist_ok=True) @@ -456,17 +446,10 @@ def _huggingface_dir(local_dir: Path) -> Path: """Return the path to the `.cache/huggingface` directory in a local directory.""" # Wrap in lru_cache to avoid overwriting the .gitignore file if called multiple times path = local_dir / ".cache" / "huggingface" - # Without long path support enabled, Windows caps directory paths at 247 characters - # (MAX_PATH minus room for an 8.3 file name), so creating the `.cache/huggingface` directory - # (and the bookkeeping files below) fails for a deep `local_dir`. - # Use the extended-length "\\?\" prefix for the filesystem operations, matching what - # `get_local_download_paths`/`get_local_upload_paths` already do for the download/upload paths. - # The un-prefixed `path` is still returned so callers keep re-deriving/prefixing as before. - target = path - if os.name == "nt": - abs_path = os.path.abspath(path) - if len(abs_path) > 247 and not abs_path.startswith("\\\\?\\"): - target = Path("\\\\?\\" + abs_path) + # Creating the `.cache/huggingface` directory (and the bookkeeping files below) fails for a deep `local_dir`, so + # the filesystem operations go through the extended-length form. The un-prefixed `path` is still returned so + # callers keep re-deriving/prefixing as before. + target = Path(as_extended_path(path, max_length=247)) target.mkdir(exist_ok=True, parents=True) # Create a CACHEDIR.TAG so backup tools can skip this directory. diff --git a/src/huggingface_hub/_snapshot_download.py b/src/huggingface_hub/_snapshot_download.py index bad4df73d1a..039762c1425 100644 --- a/src/huggingface_hub/_snapshot_download.py +++ b/src/huggingface_hub/_snapshot_download.py @@ -27,6 +27,7 @@ ) from .hf_api import HfApi, RepoFile from .utils import OfflineModeIsEnabled, filter_repo_objects, logging, validate_hf_hub_args +from .utils._paths import as_extended_path from .utils._xet_progress_reporting import ( XET_BYTES_BAR_FORMAT, XET_TRANSFER_BAR_FORMAT, @@ -693,7 +694,4 @@ def _local_file_exists(base_dir: str, path: str) -> bool: On Windows, paths longer than 255 characters must be prefixed with `\\\\?\\`, otherwise `os.path.isfile` reports an existing file as missing. """ - full_path = os.path.join(base_dir, *path.split("/")) - if os.name == "nt" and len(os.path.abspath(full_path)) > 255 and not full_path.startswith("\\\\?\\"): - full_path = "\\\\?\\" + os.path.abspath(full_path) - return os.path.isfile(full_path) + return os.path.isfile(as_extended_path(os.path.join(base_dir, *path.split("/")))) diff --git a/src/huggingface_hub/_tree_cache.py b/src/huggingface_hub/_tree_cache.py index f2d411e01a4..783f8a094ad 100644 --- a/src/huggingface_hub/_tree_cache.py +++ b/src/huggingface_hub/_tree_cache.py @@ -38,6 +38,7 @@ from dataclasses import dataclass from .utils import logging +from .utils._paths import as_extended_path from .utils._xet import is_valid_xet_hash @@ -85,25 +86,10 @@ def is_valid_tree_entries(entries: dict[str, TreeCacheEntry]) -> bool: return all(entry.xet_hash is None or is_valid_xet_hash(entry.xet_hash) for entry in entries.values()) -def _extended_length_path(absolute_path: str) -> str: - r"""Return a Windows absolute path in its extended-length form. - - A drive path becomes `\\?\C:\...`; a UNC path `\\server\share\...` becomes `\\?\UNC\server\share\...` - (prefixing a UNC path verbatim would produce an invalid `\\?\\\server` path). - """ - if absolute_path.startswith("\\\\"): - return "\\\\?\\UNC\\" + absolute_path[2:] - return "\\\\?\\" + absolute_path - - def _tree_cache_path(tree_cache_folder: str, commit_hash: str) -> str: - path = os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json") - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - # Both readers and writers go through here, so they agree on the path (and on the in-memory cache key). - if os.name == "nt" and len(os.path.abspath(path)) > 255 and not path.startswith("\\\\?\\"): - path = _extended_length_path(os.path.abspath(path)) - return path + # Both readers and writers go through here, so they agree on the path (and on the in-memory cache key), including + # on Windows where a deep `local_dir`/`cache_dir` requires the extended-length form. + return as_extended_path(os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json")) def tree_cache_folder_for_local_dir(local_dir: str) -> str: diff --git a/src/huggingface_hub/file_download.py b/src/huggingface_hub/file_download.py index 015184b8b63..c2ffdf3ec6b 100644 --- a/src/huggingface_hub/file_download.py +++ b/src/huggingface_hub/file_download.py @@ -55,6 +55,7 @@ _is_same_or_hub_host, http_stream_backoff, ) +from .utils._paths import as_extended_path from .utils._runtime import is_xet_available from .utils._xet import XetTokenType, is_valid_xet_hash, xet_connection_info_refresh_url from .utils.sha import sha_fileobj @@ -1226,21 +1227,8 @@ def _hf_hub_download_to_cache_dir( # atomically renamed into place (see `_download_to_tmp_and_move`). lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock") - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if ( - os.name == "nt" - and len(os.path.abspath(lock_path)) > 255 - and not os.path.abspath(lock_path).startswith("\\\\?\\") - ): - lock_path = "\\\\?\\" + os.path.abspath(lock_path) - - if ( - os.name == "nt" - and len(os.path.abspath(blob_path)) > 255 - and not os.path.abspath(blob_path).startswith("\\\\?\\") - ): - blob_path = "\\\\?\\" + os.path.abspath(blob_path) + lock_path = as_extended_path(lock_path) + blob_path = as_extended_path(blob_path) Path(lock_path).parent.mkdir(parents=True, exist_ok=True) @@ -1297,11 +1285,7 @@ def _hf_hub_download_to_local_dir( Method should not be called directly. Please use `hf_hub_download` instead. """ - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if os.name == "nt" and len(os.path.abspath(local_dir)) > 255: - local_dir = "\\\\?\\" + os.path.abspath(local_dir) - local_dir = Path(local_dir) + local_dir = Path(as_extended_path(local_dir)) paths = get_local_download_paths(local_dir=local_dir, filename=filename) local_metadata = read_download_metadata(local_dir=local_dir, filename=filename) diff --git a/src/huggingface_hub/utils/_paths.py b/src/huggingface_hub/utils/_paths.py index f1420f98642..87e8d212de0 100644 --- a/src/huggingface_hub/utils/_paths.py +++ b/src/huggingface_hub/utils/_paths.py @@ -13,6 +13,7 @@ # limitations under the License. """Contains utilities to handle paths in Huggingface Hub.""" +import os from collections.abc import Callable, Generator, Iterable from fnmatch import fnmatchcase from pathlib import Path @@ -36,6 +37,33 @@ FORBIDDEN_FOLDERS = [".git", ".cache"] +def as_extended_path(path: str | Path, max_length: int = 255) -> str: + r"""Return `path` in its Windows extended-length form if it is too long, unchanged otherwise. + + Some Windows versions do not allow for paths longer than 255 characters (247 for directories, i.e. MAX_PATH minus + room for an 8.3 file name). In this case, we must specify them as extended paths by using the `\\?\` prefix, which + only works on absolute paths. Network shares take the `\\?\UNC\server\share\...` form: prefixing them verbatim + would produce an invalid `\\?\\\server\...` path. + + Args: + path (`str` or `Path`): + The path to convert. Returned unchanged on non-Windows platforms, if it is short enough, or if it is + already an extended path. + max_length (`int`, *optional*): + Length above which the path must be converted. Defaults to 255, the limit for files. Pass 247 for paths + whose parent directories are created by the caller. + """ + path = str(path) + if os.name != "nt": + return path + absolute_path = os.path.abspath(path) + if len(absolute_path) <= max_length or absolute_path.startswith("\\\\?\\"): + return path + if absolute_path.startswith("\\\\"): # UNC share: `\\server\share\...` => `\\?\UNC\server\share\...` + return "\\\\?\\UNC\\" + absolute_path[2:] + return "\\\\?\\" + absolute_path + + def filter_repo_objects( items: Iterable[T], *, diff --git a/tests/test_tree_cache.py b/tests/test_tree_cache.py index 2d63ee8e540..d5510bef3c0 100644 --- a/tests/test_tree_cache.py +++ b/tests/test_tree_cache.py @@ -124,25 +124,13 @@ def test_in_memory_cache_memoizes_first_read(self, tmp_path: Path): mock_read.assert_not_called() @pytest.mark.skipif(os.name != "nt", reason="Windows-specific test.") - @pytest.mark.parametrize("tree_cache_path_len", [260, 300]) - def test_round_trip_in_deep_folder(self, tmp_path: Path, tree_cache_path_len: int): - r"""A deep download target must still get a usable tree cache on Windows. - - Without long path support enabled, Windows caps file paths at 255 characters (and directory - paths at 247), so for a deep ``local_dir`` (or ``cache_dir``) writing - ``/trees/.json`` failed. `write_tree_cache` swallows that failure and the - next read simply misses, so the cache was silently disabled and every download re-fetched the - tree listing. Both boundary cases are covered: 260 (only the JSON file exceeds the limit, the - ``trees/`` folder still fits) and 300 (the ``trees/`` folder exceeds it too, so the `makedirs` - needs the prefix as well). - """ - # Pad the folder so that `/trees/.json` is `tree_cache_path_len` chars long. + def test_round_trip_in_deep_folder(self, tmp_path: Path): + """Regression test for https://github.com/huggingface/huggingface_hub/issues/4895.""" + # Pad `local_dir` so that `/trees/.json` exceeds the Windows path limit. # Use the extended-length prefix here since a plain mkdir of such a path would itself fail. - relative_depth = len(str(Path(".cache") / "huggingface" / "trees" / f"{COMMIT_HASH}.json")) - local_dir = tmp_path / ("d" * max(1, tree_cache_path_len - len(str(tmp_path)) - relative_depth - 2)) + local_dir = tmp_path / ("d" * 200) os.makedirs("\\\\?\\" + os.path.abspath(local_dir), exist_ok=True) folder = tree_cache_folder_for_local_dir(str(local_dir)) - assert len(str(Path(folder) / "trees" / f"{COMMIT_HASH}.json")) >= tree_cache_path_len write_tree_cache(folder, COMMIT_HASH, _entries()) @@ -150,27 +138,6 @@ def test_round_trip_in_deep_folder(self, tmp_path: Path, tree_cache_path_len: in _IN_MEMORY_TREE_CACHE.pop(_tree_cache_path(folder, COMMIT_HASH), None) assert read_tree_cache(folder, COMMIT_HASH) == _entries() - @pytest.mark.skipif(os.name != "nt", reason="Windows-specific test.") - def test_extended_length_prefix_for_drive_and_unc_paths(self): - r"""Long drive paths get the `\\?\` prefix, long UNC paths the `\\?\UNC\` form, short paths are untouched.""" - long_drive_folder = "C:\\" + "d" * 300 - assert _tree_cache_path(long_drive_folder, COMMIT_HASH) == ( - "\\\\?\\" + long_drive_folder + "\\trees\\" + f"{COMMIT_HASH}.json" - ) - - # A UNC share must not become `\\?\\\server\share` (invalid); it must become `\\?\UNC\server\share`. - long_unc_folder = "\\\\server\\share\\" + "d" * 300 - assert _tree_cache_path(long_unc_folder, COMMIT_HASH) == ( - "\\\\?\\UNC\\server\\share\\" + "d" * 300 + "\\trees\\" + f"{COMMIT_HASH}.json" - ) - - # Already-prefixed and short paths come back as-is: no double prefix, no behavior change. - already_prefixed = "\\\\?\\C:\\" + "d" * 300 - assert _tree_cache_path(already_prefixed, COMMIT_HASH).count("\\\\?\\") == 1 - assert _tree_cache_path("C:\\models\\mymodel", COMMIT_HASH) == os.path.join( - "C:\\models\\mymodel", "trees", f"{COMMIT_HASH}.json" - ) - @pytest.fixture def tree_cache_folder(tmp_path: Path):