diff --git a/src/huggingface_hub/_local_folder.py b/src/huggingface_hub/_local_folder.py index 8af934ca56e..4863292c574 100644 --- a/src/huggingface_hub/_local_folder.py +++ b/src/huggingface_hub/_local_folder.py @@ -57,6 +57,7 @@ from pathlib import Path, PurePosixPath, PureWindowsPath from .utils import WeakFileLock +from .utils._paths import as_extended_path logger = logging.getLogger(__name__) @@ -92,12 +93,7 @@ class LocalDownloadFilePaths: def incomplete_path(self, etag: str) -> Path: """Return the path where a file will be temporarily downloaded before being moved to `file_path`.""" path = self.metadata_path.parent / f"{_short_hash(self.metadata_path.name)}.{etag}.incomplete" - resolved_path = str(path.resolve()) - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if os.name == "nt" and len(resolved_path) > 255 and not resolved_path.startswith("\\\\?\\"): - path = Path("\\\\?\\" + resolved_path) - return path + return Path(as_extended_path(path)) @dataclass(frozen=True) @@ -253,13 +249,10 @@ def get_local_download_paths(local_dir: Path, filename: str) -> LocalDownloadFil metadata_path = _huggingface_dir(local_dir) / "download" / f"{sanitized_filename}.metadata" lock_path = metadata_path.with_suffix(".lock") - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix - if os.name == "nt": - if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: - file_path = Path("\\\\?\\" + os.path.abspath(file_path)) - lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) - metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) + # Parent directories are created below => use the stricter Windows limit for directories. + file_path = Path(as_extended_path(file_path, max_length=247)) + lock_path = Path(as_extended_path(lock_path, max_length=247)) + metadata_path = Path(as_extended_path(metadata_path, max_length=247)) file_path.parent.mkdir(parents=True, exist_ok=True) metadata_path.parent.mkdir(parents=True, exist_ok=True) @@ -288,13 +281,10 @@ def get_local_upload_paths(local_dir: Path, filename: str) -> LocalUploadFilePat metadata_path = _huggingface_dir(local_dir) / "upload" / f"{sanitized_filename}.metadata" lock_path = metadata_path.with_suffix(".lock") - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix - if os.name == "nt": - if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: - file_path = Path("\\\\?\\" + os.path.abspath(file_path)) - lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) - metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) + # Parent directories are created below => use the stricter Windows limit for directories. + file_path = Path(as_extended_path(file_path, max_length=247)) + lock_path = Path(as_extended_path(lock_path, max_length=247)) + metadata_path = Path(as_extended_path(metadata_path, max_length=247)) file_path.parent.mkdir(parents=True, exist_ok=True) metadata_path.parent.mkdir(parents=True, exist_ok=True) @@ -456,17 +446,10 @@ def _huggingface_dir(local_dir: Path) -> Path: """Return the path to the `.cache/huggingface` directory in a local directory.""" # Wrap in lru_cache to avoid overwriting the .gitignore file if called multiple times path = local_dir / ".cache" / "huggingface" - # Without long path support enabled, Windows caps directory paths at 247 characters - # (MAX_PATH minus room for an 8.3 file name), so creating the `.cache/huggingface` directory - # (and the bookkeeping files below) fails for a deep `local_dir`. - # Use the extended-length "\\?\" prefix for the filesystem operations, matching what - # `get_local_download_paths`/`get_local_upload_paths` already do for the download/upload paths. - # The un-prefixed `path` is still returned so callers keep re-deriving/prefixing as before. - target = path - if os.name == "nt": - abs_path = os.path.abspath(path) - if len(abs_path) > 247 and not abs_path.startswith("\\\\?\\"): - target = Path("\\\\?\\" + abs_path) + # Creating the `.cache/huggingface` directory (and the bookkeeping files below) fails for a deep `local_dir`, so + # the filesystem operations go through the extended-length form. The un-prefixed `path` is still returned so + # callers keep re-deriving/prefixing as before. + target = Path(as_extended_path(path, max_length=247)) target.mkdir(exist_ok=True, parents=True) # Create a CACHEDIR.TAG so backup tools can skip this directory. diff --git a/src/huggingface_hub/_snapshot_download.py b/src/huggingface_hub/_snapshot_download.py index bad4df73d1a..039762c1425 100644 --- a/src/huggingface_hub/_snapshot_download.py +++ b/src/huggingface_hub/_snapshot_download.py @@ -27,6 +27,7 @@ ) from .hf_api import HfApi, RepoFile from .utils import OfflineModeIsEnabled, filter_repo_objects, logging, validate_hf_hub_args +from .utils._paths import as_extended_path from .utils._xet_progress_reporting import ( XET_BYTES_BAR_FORMAT, XET_TRANSFER_BAR_FORMAT, @@ -693,7 +694,4 @@ def _local_file_exists(base_dir: str, path: str) -> bool: On Windows, paths longer than 255 characters must be prefixed with `\\\\?\\`, otherwise `os.path.isfile` reports an existing file as missing. """ - full_path = os.path.join(base_dir, *path.split("/")) - if os.name == "nt" and len(os.path.abspath(full_path)) > 255 and not full_path.startswith("\\\\?\\"): - full_path = "\\\\?\\" + os.path.abspath(full_path) - return os.path.isfile(full_path) + return os.path.isfile(as_extended_path(os.path.join(base_dir, *path.split("/")))) diff --git a/src/huggingface_hub/_tree_cache.py b/src/huggingface_hub/_tree_cache.py index 7026bc41939..783f8a094ad 100644 --- a/src/huggingface_hub/_tree_cache.py +++ b/src/huggingface_hub/_tree_cache.py @@ -38,6 +38,7 @@ from dataclasses import dataclass from .utils import logging +from .utils._paths import as_extended_path from .utils._xet import is_valid_xet_hash @@ -86,7 +87,9 @@ def is_valid_tree_entries(entries: dict[str, TreeCacheEntry]) -> bool: def _tree_cache_path(tree_cache_folder: str, commit_hash: str) -> str: - return os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json") + # Both readers and writers go through here, so they agree on the path (and on the in-memory cache key), including + # on Windows where a deep `local_dir`/`cache_dir` requires the extended-length form. + return as_extended_path(os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json")) def tree_cache_folder_for_local_dir(local_dir: str) -> str: diff --git a/src/huggingface_hub/file_download.py b/src/huggingface_hub/file_download.py index 015184b8b63..c2ffdf3ec6b 100644 --- a/src/huggingface_hub/file_download.py +++ b/src/huggingface_hub/file_download.py @@ -55,6 +55,7 @@ _is_same_or_hub_host, http_stream_backoff, ) +from .utils._paths import as_extended_path from .utils._runtime import is_xet_available from .utils._xet import XetTokenType, is_valid_xet_hash, xet_connection_info_refresh_url from .utils.sha import sha_fileobj @@ -1226,21 +1227,8 @@ def _hf_hub_download_to_cache_dir( # atomically renamed into place (see `_download_to_tmp_and_move`). lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock") - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if ( - os.name == "nt" - and len(os.path.abspath(lock_path)) > 255 - and not os.path.abspath(lock_path).startswith("\\\\?\\") - ): - lock_path = "\\\\?\\" + os.path.abspath(lock_path) - - if ( - os.name == "nt" - and len(os.path.abspath(blob_path)) > 255 - and not os.path.abspath(blob_path).startswith("\\\\?\\") - ): - blob_path = "\\\\?\\" + os.path.abspath(blob_path) + lock_path = as_extended_path(lock_path) + blob_path = as_extended_path(blob_path) Path(lock_path).parent.mkdir(parents=True, exist_ok=True) @@ -1297,11 +1285,7 @@ def _hf_hub_download_to_local_dir( Method should not be called directly. Please use `hf_hub_download` instead. """ - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if os.name == "nt" and len(os.path.abspath(local_dir)) > 255: - local_dir = "\\\\?\\" + os.path.abspath(local_dir) - local_dir = Path(local_dir) + local_dir = Path(as_extended_path(local_dir)) paths = get_local_download_paths(local_dir=local_dir, filename=filename) local_metadata = read_download_metadata(local_dir=local_dir, filename=filename) diff --git a/src/huggingface_hub/utils/_paths.py b/src/huggingface_hub/utils/_paths.py index f1420f98642..87e8d212de0 100644 --- a/src/huggingface_hub/utils/_paths.py +++ b/src/huggingface_hub/utils/_paths.py @@ -13,6 +13,7 @@ # limitations under the License. """Contains utilities to handle paths in Huggingface Hub.""" +import os from collections.abc import Callable, Generator, Iterable from fnmatch import fnmatchcase from pathlib import Path @@ -36,6 +37,33 @@ FORBIDDEN_FOLDERS = [".git", ".cache"] +def as_extended_path(path: str | Path, max_length: int = 255) -> str: + r"""Return `path` in its Windows extended-length form if it is too long, unchanged otherwise. + + Some Windows versions do not allow for paths longer than 255 characters (247 for directories, i.e. MAX_PATH minus + room for an 8.3 file name). In this case, we must specify them as extended paths by using the `\\?\` prefix, which + only works on absolute paths. Network shares take the `\\?\UNC\server\share\...` form: prefixing them verbatim + would produce an invalid `\\?\\\server\...` path. + + Args: + path (`str` or `Path`): + The path to convert. Returned unchanged on non-Windows platforms, if it is short enough, or if it is + already an extended path. + max_length (`int`, *optional*): + Length above which the path must be converted. Defaults to 255, the limit for files. Pass 247 for paths + whose parent directories are created by the caller. + """ + path = str(path) + if os.name != "nt": + return path + absolute_path = os.path.abspath(path) + if len(absolute_path) <= max_length or absolute_path.startswith("\\\\?\\"): + return path + if absolute_path.startswith("\\\\"): # UNC share: `\\server\share\...` => `\\?\UNC\server\share\...` + return "\\\\?\\UNC\\" + absolute_path[2:] + return "\\\\?\\" + absolute_path + + def filter_repo_objects( items: Iterable[T], *, diff --git a/tests/test_tree_cache.py b/tests/test_tree_cache.py index c6516b94b1e..d5510bef3c0 100644 --- a/tests/test_tree_cache.py +++ b/tests/test_tree_cache.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +import os from pathlib import Path from unittest.mock import patch @@ -22,6 +23,7 @@ _IN_MEMORY_TREE_CACHE, TREE_CACHE_FORMAT_VERSION, TreeCacheEntry, + _tree_cache_path, read_tree_cache, tree_cache_folder_for_local_dir, write_tree_cache, @@ -121,6 +123,21 @@ def test_in_memory_cache_memoizes_first_read(self, tmp_path: Path): assert read_tree_cache(str(tmp_path), COMMIT_HASH) == _entries() mock_read.assert_not_called() + @pytest.mark.skipif(os.name != "nt", reason="Windows-specific test.") + def test_round_trip_in_deep_folder(self, tmp_path: Path): + """Regression test for https://github.com/huggingface/huggingface_hub/issues/4895.""" + # Pad `local_dir` so that `/trees/.json` exceeds the Windows path limit. + # Use the extended-length prefix here since a plain mkdir of such a path would itself fail. + local_dir = tmp_path / ("d" * 200) + os.makedirs("\\\\?\\" + os.path.abspath(local_dir), exist_ok=True) + folder = tree_cache_folder_for_local_dir(str(local_dir)) + + write_tree_cache(folder, COMMIT_HASH, _entries()) + + # Drop the in-memory entry seeded by the write, so the read has to go through the disk. + _IN_MEMORY_TREE_CACHE.pop(_tree_cache_path(folder, COMMIT_HASH), None) + assert read_tree_cache(folder, COMMIT_HASH) == _entries() + @pytest.fixture def tree_cache_folder(tmp_path: Path):