Skip to content
6 changes: 3 additions & 3 deletions src/huggingface_hub/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ def _as_int(value: str | None) -> int | None:
ENDPOINT = _HF_DEFAULT_STAGING_ENDPOINT
HUGGINGFACE_CO_URL_TEMPLATE = _HF_DEFAULT_STAGING_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"

# Hosts whose web URLs can be parsed into a ``hf://`` URI (see ``huggingface_hub/utils/_hf_uris.py``).
# Includes the public Hub host and its ``hf.co`` short domain, the staging host, and the host of the
# currently configured ``ENDPOINT`` so that self-hosted / staging endpoints work too.
# Hosts considered to be Hugging Face Hub endpoints: the public Hub host, its ``hf.co`` short domain, the staging
# host, and the host of the configured ``ENDPOINT``. Used to parse web URLs into ``hf://`` URIs and to decide which
# redirects to follow when resolving files. The auth header is forwarded to these hosts: only add trusted ones.
HF_URL_HOSTS: frozenset[str] = frozenset(
{"hf.co"}
| {
Expand Down
27 changes: 17 additions & 10 deletions src/huggingface_hub/file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from dataclasses import dataclass
from pathlib import Path
from typing import Any, BinaryIO, Literal, NoReturn, overload
from urllib.parse import quote, urlparse
from urllib.parse import quote

import httpx
from tqdm.auto import tqdm as base_tqdm
Expand Down Expand Up @@ -50,7 +50,8 @@
_DEFAULT_RETRY_ON_EXCEPTIONS,
_DEFAULT_RETRY_ON_STATUS_CODES,
_adjust_range_header,
_httpx_follow_relative_redirects_with_backoff,
_httpx_follow_hub_redirects_with_backoff,
_is_same_or_hub_host,
http_stream_backoff,
)
from .utils._runtime import is_xet_available
Expand Down Expand Up @@ -1618,7 +1619,7 @@ def get_hf_file_metadata(
hf_headers["Accept-Encoding"] = "identity" # prevent any compression => we want to know the real size of the file

# Retrieve metadata
response = _httpx_follow_relative_redirects_with_backoff(
response = _httpx_follow_hub_redirects_with_backoff(
method="HEAD", url=url, headers=hf_headers, timeout=timeout, retry_on_errors=retry_on_errors
)
hf_raise_for_status(response)
Expand Down Expand Up @@ -1740,9 +1741,10 @@ def _get_metadata_or_catch_error(
commit_hash = metadata.commit_hash
if commit_hash is None:
raise FileMetadataError(
"Distant resource does not seem to be on huggingface.co. It is possible that a configuration issue"
" prevents you from downloading resources from https://huggingface.co. Please check your firewall"
" and proxy settings and make sure your SSL certificates are updated."
f"Response from {url} is missing the '{constants.HUGGINGFACE_HEADER_X_REPO_COMMIT}' header, so it"
" does not seem to be served by a Hugging Face Hub endpoint. If HF_ENDPOINT is set, check that it"
" points to a Hub-compatible endpoint. Otherwise, check your firewall and proxy settings and make"
" sure your SSL certificates are updated."
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
)

# Etag must exist
Expand All @@ -1764,12 +1766,12 @@ def _get_metadata_or_catch_error(
# and ensure we download the exact atomic version even if it changed
# between the HEAD and the GET (unlikely, but hey).
#
# If url domain is different => we are downloading from a CDN => url is signed => don't send auth
# If url domain is the same => redirect due to repo rename AND downloading a regular file => keep auth
# If the final location is on a Hub host (same host, or e.g. huggingface.co reached through an
# HF_ENDPOINT mirror redirect) => keep auth. Otherwise it's a signed CDN url => don't send auth.
if xet_file_data is None and url != metadata.location:
url_to_download = metadata.location
if urlparse(url).netloc != urlparse(metadata.location).netloc:
# Remove authorization header when downloading a LFS blob
if not _is_same_or_hub_host(url, metadata.location):
# Remove authorization header when downloading a LFS blob from a CDN
headers.pop("authorization", None)
except httpx.ProxyError:
# Actually raise on proxy error
Expand Down Expand Up @@ -1903,6 +1905,11 @@ def _raise_on_head_call_error(head_call_error: Exception, force_download: bool,
# Repo not found or gated => let's raise the actual error
# Unauthorized => likely a token issue => let's raise the actual error
raise head_call_error
elif isinstance(head_call_error, FileMetadataError):
# The call succeeded but the response lacked the metadata we need => a configuration issue, not connectivity.
raise LocalEntryNotFoundError(
f"{head_call_error} We also cannot find the requested files in the local cache."
) from head_call_error
else:
# Otherwise: most likely a connection issue or Hub downtime => let's warn the user
raise LocalEntryNotFoundError(
Expand Down
4 changes: 2 additions & 2 deletions src/huggingface_hub/hf_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@
from .utils import tqdm as hf_tqdm
from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab
from .utils._deprecation import _deprecate_arguments, _deprecate_method
from .utils._http import _httpx_follow_relative_redirects_with_backoff
from .utils._http import _httpx_follow_hub_redirects_with_backoff
from .utils._runtime import is_xet_available
from .utils._typing import CallableT
from .utils._verification import collect_local_files, resolve_local_root, verify_maps
Expand Down Expand Up @@ -14735,7 +14735,7 @@ def get_bucket_file_metadata(
42000
```
"""
response = _httpx_follow_relative_redirects_with_backoff(
response = _httpx_follow_hub_redirects_with_backoff(
"HEAD",
f"{self.endpoint}/buckets/{bucket_id}/resolve/{quote(remote_path, safe='')}",
headers=self._build_hf_headers(token=token),
Expand Down
40 changes: 25 additions & 15 deletions src/huggingface_hub/utils/_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from dataclasses import dataclass
from shlex import quote
from typing import Any, TypeVar
from urllib.parse import urlparse
from urllib.parse import urljoin, urlparse

import httpx

Expand Down Expand Up @@ -691,14 +691,18 @@ def http_stream_backoff(
)


def _httpx_follow_relative_redirects_with_backoff(
_MAX_REDIRECTS = 20 # same bound as httpx's default max_redirects


def _httpx_follow_hub_redirects_with_backoff(
method: HTTP_METHOD_T, url: str, *, retry_on_errors: bool = False, **httpx_kwargs
) -> httpx.Response:
"""Perform an HTTP request with backoff and follow relative redirects only.
"""Perform an HTTP request with backoff, following redirects that stay on the Hub.

Used to fetch HEAD /resolve on repo or bucket files.

This is useful to follow a redirection to a renamed repository without following redirection to a CDN.
Redirects to the same host or another Hub host are followed. Redirects to any other host (CDN, storage
bucket) are not: the file metadata is on the redirect response itself and the auth header must not leave the Hub.

A backoff mechanism retries the HTTP call on errors (429, 5xx, timeout, network errors).

Expand All @@ -718,7 +722,7 @@ def _httpx_follow_relative_redirects_with_backoff(
{} if retry_on_errors else {"retry_on_exceptions": (), "retry_on_status_codes": ()}
)

while True:
for _ in range(_MAX_REDIRECTS):
response = http_backoff(
method=method,
url=url,
Expand All @@ -728,18 +732,24 @@ def _httpx_follow_relative_redirects_with_backoff(
)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
hf_raise_for_status(response)

# Check if response is a relative redirect
if 300 <= response.status_code <= 399:
parsed_target = urlparse(response.headers["Location"])
if parsed_target.netloc == "":
# Relative redirect -> update URL and retry
url = urlparse(url)._replace(path=parsed_target.path).geturl()
continue
if not response.has_redirect_location:
return response

target = urljoin(url, response.headers["Location"])

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL (again?) urljoin behavior is so confusing 😵

https://stackoverflow.com/questions/10893374/python-confusions-with-urljoin

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lol Claude sent me to the same stackoverflow page because i didn't understand much in urljoin behavior

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

well 14 years ago and still top 1 google result so we are not the only ones^^

if not _is_same_or_hub_host(url, target):
# Redirect to a CDN or storage host (signed URL): the file metadata is carried by this very response,
# and the authorization header must not be forwarded off the Hub => stop here.
return response

url = target

raise httpx.TooManyRedirects(f"Exceeded {_MAX_REDIRECTS} redirects while resolving '{url}'.")

# Break if no relative redirect
break

return response
def _is_same_or_hub_host(url: str, target: str) -> bool:
"""Whether `target` is served by the same host as `url`, or by a known Hub host."""
target_host = (urlparse(target).hostname or "").lower()
return target_host == (urlparse(url).hostname or "").lower() or target_host in constants.HF_URL_HOSTS
Comment thread
hanouticelina marked this conversation as resolved.
Resolved


def fix_hf_endpoint_in_url(url: str, endpoint: str | None) -> str:
Expand Down
Loading