Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 14 additions & 31 deletions src/huggingface_hub/_local_folder.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
from pathlib import Path, PurePosixPath, PureWindowsPath

from .utils import WeakFileLock
from .utils._paths import as_extended_path


logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -92,12 +93,7 @@ class LocalDownloadFilePaths:
def incomplete_path(self, etag: str) -> Path:
"""Return the path where a file will be temporarily downloaded before being moved to `file_path`."""
path = self.metadata_path.parent / f"{_short_hash(self.metadata_path.name)}.{etag}.incomplete"
resolved_path = str(path.resolve())
# Some Windows versions do not allow for paths longer than 255 characters.
# In this case, we must specify it as an extended path by using the "\\?\" prefix.
if os.name == "nt" and len(resolved_path) > 255 and not resolved_path.startswith("\\\\?\\"):
path = Path("\\\\?\\" + resolved_path)
return path
return Path(as_extended_path(path))


@dataclass(frozen=True)
Expand Down Expand Up @@ -253,13 +249,10 @@ def get_local_download_paths(local_dir: Path, filename: str) -> LocalDownloadFil
metadata_path = _huggingface_dir(local_dir) / "download" / f"{sanitized_filename}.metadata"
lock_path = metadata_path.with_suffix(".lock")

# Some Windows versions do not allow for paths longer than 255 characters.
# In this case, we must specify it as an extended path by using the "\\?\" prefix
if os.name == "nt":
if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255:
file_path = Path("\\\\?\\" + os.path.abspath(file_path))
lock_path = Path("\\\\?\\" + os.path.abspath(lock_path))
metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path))
# Parent directories are created below => use the stricter Windows limit for directories.
file_path = Path(as_extended_path(file_path, max_length=247))
lock_path = Path(as_extended_path(lock_path, max_length=247))
metadata_path = Path(as_extended_path(metadata_path, max_length=247))

file_path.parent.mkdir(parents=True, exist_ok=True)
metadata_path.parent.mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -288,13 +281,10 @@ def get_local_upload_paths(local_dir: Path, filename: str) -> LocalUploadFilePat
metadata_path = _huggingface_dir(local_dir) / "upload" / f"{sanitized_filename}.metadata"
lock_path = metadata_path.with_suffix(".lock")

# Some Windows versions do not allow for paths longer than 255 characters.
# In this case, we must specify it as an extended path by using the "\\?\" prefix
if os.name == "nt":
if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255:
file_path = Path("\\\\?\\" + os.path.abspath(file_path))
lock_path = Path("\\\\?\\" + os.path.abspath(lock_path))
metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path))
# Parent directories are created below => use the stricter Windows limit for directories.
file_path = Path(as_extended_path(file_path, max_length=247))
lock_path = Path(as_extended_path(lock_path, max_length=247))
metadata_path = Path(as_extended_path(metadata_path, max_length=247))

file_path.parent.mkdir(parents=True, exist_ok=True)
metadata_path.parent.mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -456,17 +446,10 @@ def _huggingface_dir(local_dir: Path) -> Path:
"""Return the path to the `.cache/huggingface` directory in a local directory."""
# Wrap in lru_cache to avoid overwriting the .gitignore file if called multiple times
path = local_dir / ".cache" / "huggingface"
# Without long path support enabled, Windows caps directory paths at 247 characters
# (MAX_PATH minus room for an 8.3 file name), so creating the `.cache/huggingface` directory
# (and the bookkeeping files below) fails for a deep `local_dir`.
# Use the extended-length "\\?\" prefix for the filesystem operations, matching what
# `get_local_download_paths`/`get_local_upload_paths` already do for the download/upload paths.
# The un-prefixed `path` is still returned so callers keep re-deriving/prefixing as before.
target = path
if os.name == "nt":
abs_path = os.path.abspath(path)
if len(abs_path) > 247 and not abs_path.startswith("\\\\?\\"):
target = Path("\\\\?\\" + abs_path)
# Creating the `.cache/huggingface` directory (and the bookkeeping files below) fails for a deep `local_dir`, so
# the filesystem operations go through the extended-length form. The un-prefixed `path` is still returned so
# callers keep re-deriving/prefixing as before.
target = Path(as_extended_path(path, max_length=247))
target.mkdir(exist_ok=True, parents=True)

# Create a CACHEDIR.TAG so backup tools can skip this directory.
Expand Down
6 changes: 2 additions & 4 deletions src/huggingface_hub/_snapshot_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
)
from .hf_api import HfApi, RepoFile
from .utils import OfflineModeIsEnabled, filter_repo_objects, logging, validate_hf_hub_args
from .utils._paths import as_extended_path
from .utils._xet_progress_reporting import (
XET_BYTES_BAR_FORMAT,
XET_TRANSFER_BAR_FORMAT,
Expand Down Expand Up @@ -693,7 +694,4 @@ def _local_file_exists(base_dir: str, path: str) -> bool:
On Windows, paths longer than 255 characters must be prefixed with `\\\\?\\`, otherwise `os.path.isfile` reports an
existing file as missing.
"""
full_path = os.path.join(base_dir, *path.split("/"))
if os.name == "nt" and len(os.path.abspath(full_path)) > 255 and not full_path.startswith("\\\\?\\"):
full_path = "\\\\?\\" + os.path.abspath(full_path)
return os.path.isfile(full_path)
return os.path.isfile(as_extended_path(os.path.join(base_dir, *path.split("/"))))
5 changes: 4 additions & 1 deletion src/huggingface_hub/_tree_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from dataclasses import dataclass

from .utils import logging
from .utils._paths import as_extended_path
from .utils._xet import is_valid_xet_hash


Expand Down Expand Up @@ -86,7 +87,9 @@ def is_valid_tree_entries(entries: dict[str, TreeCacheEntry]) -> bool:


def _tree_cache_path(tree_cache_folder: str, commit_hash: str) -> str:
return os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json")
# Both readers and writers go through here, so they agree on the path (and on the in-memory cache key), including
# on Windows where a deep `local_dir`/`cache_dir` requires the extended-length form.
return as_extended_path(os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json"))


def tree_cache_folder_for_local_dir(local_dir: str) -> str:
Expand Down
24 changes: 4 additions & 20 deletions src/huggingface_hub/file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
_is_same_or_hub_host,
http_stream_backoff,
)
from .utils._paths import as_extended_path
from .utils._runtime import is_xet_available
from .utils._xet import XetTokenType, is_valid_xet_hash, xet_connection_info_refresh_url
from .utils.sha import sha_fileobj
Expand Down Expand Up @@ -1226,21 +1227,8 @@ def _hf_hub_download_to_cache_dir(
# atomically renamed into place (see `_download_to_tmp_and_move`).
lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock")

# Some Windows versions do not allow for paths longer than 255 characters.
# In this case, we must specify it as an extended path by using the "\\?\" prefix.
if (
os.name == "nt"
and len(os.path.abspath(lock_path)) > 255
and not os.path.abspath(lock_path).startswith("\\\\?\\")
):
lock_path = "\\\\?\\" + os.path.abspath(lock_path)

if (
os.name == "nt"
and len(os.path.abspath(blob_path)) > 255
and not os.path.abspath(blob_path).startswith("\\\\?\\")
):
blob_path = "\\\\?\\" + os.path.abspath(blob_path)
lock_path = as_extended_path(lock_path)
blob_path = as_extended_path(blob_path)

Path(lock_path).parent.mkdir(parents=True, exist_ok=True)

Expand Down Expand Up @@ -1297,11 +1285,7 @@ def _hf_hub_download_to_local_dir(

Method should not be called directly. Please use `hf_hub_download` instead.
"""
# Some Windows versions do not allow for paths longer than 255 characters.
# In this case, we must specify it as an extended path by using the "\\?\" prefix.
if os.name == "nt" and len(os.path.abspath(local_dir)) > 255:
local_dir = "\\\\?\\" + os.path.abspath(local_dir)
local_dir = Path(local_dir)
local_dir = Path(as_extended_path(local_dir))
paths = get_local_download_paths(local_dir=local_dir, filename=filename)
local_metadata = read_download_metadata(local_dir=local_dir, filename=filename)

Expand Down
28 changes: 28 additions & 0 deletions src/huggingface_hub/utils/_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.
"""Contains utilities to handle paths in Huggingface Hub."""

import os
from collections.abc import Callable, Generator, Iterable
from fnmatch import fnmatchcase
from pathlib import Path
Expand All @@ -36,6 +37,33 @@
FORBIDDEN_FOLDERS = [".git", ".cache"]


def as_extended_path(path: str | Path, max_length: int = 255) -> str:
r"""Return `path` in its Windows extended-length form if it is too long, unchanged otherwise.

Some Windows versions do not allow for paths longer than 255 characters (247 for directories, i.e. MAX_PATH minus
room for an 8.3 file name). In this case, we must specify them as extended paths by using the `\\?\` prefix, which
only works on absolute paths. Network shares take the `\\?\UNC\server\share\...` form: prefixing them verbatim
would produce an invalid `\\?\\\server\...` path.

Args:
path (`str` or `Path`):
The path to convert. Returned unchanged on non-Windows platforms, if it is short enough, or if it is
already an extended path.
max_length (`int`, *optional*):
Length above which the path must be converted. Defaults to 255, the limit for files. Pass 247 for paths
whose parent directories are created by the caller.
"""
path = str(path)
if os.name != "nt":
return path
absolute_path = os.path.abspath(path)
if len(absolute_path) <= max_length or absolute_path.startswith("\\\\?\\"):
return path
if absolute_path.startswith("\\\\"): # UNC share: `\\server\share\...` => `\\?\UNC\server\share\...`
return "\\\\?\\UNC\\" + absolute_path[2:]
return "\\\\?\\" + absolute_path


def filter_repo_objects(
items: Iterable[T],
*,
Expand Down
17 changes: 17 additions & 0 deletions tests/test_tree_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from pathlib import Path
from unittest.mock import patch

Expand All @@ -22,6 +23,7 @@
_IN_MEMORY_TREE_CACHE,
TREE_CACHE_FORMAT_VERSION,
TreeCacheEntry,
_tree_cache_path,
read_tree_cache,
tree_cache_folder_for_local_dir,
write_tree_cache,
Expand Down Expand Up @@ -121,6 +123,21 @@ def test_in_memory_cache_memoizes_first_read(self, tmp_path: Path):
assert read_tree_cache(str(tmp_path), COMMIT_HASH) == _entries()
mock_read.assert_not_called()

@pytest.mark.skipif(os.name != "nt", reason="Windows-specific test.")
def test_round_trip_in_deep_folder(self, tmp_path: Path):
"""Regression test for https://github.com/huggingface/huggingface_hub/issues/4895."""
# Pad `local_dir` so that `<folder>/trees/<commit_hash>.json` exceeds the Windows path limit.
# Use the extended-length prefix here since a plain mkdir of such a path would itself fail.
local_dir = tmp_path / ("d" * 200)
os.makedirs("\\\\?\\" + os.path.abspath(local_dir), exist_ok=True)
folder = tree_cache_folder_for_local_dir(str(local_dir))

write_tree_cache(folder, COMMIT_HASH, _entries())

# Drop the in-memory entry seeded by the write, so the read has to go through the disk.
_IN_MEMORY_TREE_CACHE.pop(_tree_cache_path(folder, COMMIT_HASH), None)
assert read_tree_cache(folder, COMMIT_HASH) == _entries()


@pytest.fixture
def tree_cache_folder(tmp_path: Path):
Expand Down
Loading