Skip to content
9 changes: 6 additions & 3 deletions src/huggingface_hub/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,12 @@ def _as_int(value: str | None) -> int | None:
ENDPOINT = _HF_DEFAULT_STAGING_ENDPOINT
HUGGINGFACE_CO_URL_TEMPLATE = _HF_DEFAULT_STAGING_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"

# Hosts whose web URLs can be parsed into a ``hf://`` URI (see ``huggingface_hub/utils/_hf_uris.py``).
# Includes the public Hub host and its ``hf.co`` short domain, the staging host, and the host of the
# currently configured ``ENDPOINT`` so that self-hosted / staging endpoints work too.
# Hosts we consider to be Hugging Face Hub endpoints. Includes the public Hub host and its ``hf.co`` short
# domain, the staging host, and the host of the currently configured ``ENDPOINT`` so that self-hosted / staging
# endpoints work too.
# Used to parse web URLs into ``hf://`` URIs (see ``huggingface_hub/utils/_hf_uris.py``) and to decide which
# redirects to follow while resolving files (see ``huggingface_hub/utils/_http.py``). The latter forwards the
# authorization header to these hosts, so only add a host that is trusted with the user's token.
HF_URL_HOSTS: frozenset[str] = frozenset(
{"hf.co"}
| {
Expand Down
16 changes: 11 additions & 5 deletions src/huggingface_hub/file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
_DEFAULT_RETRY_ON_EXCEPTIONS,
_DEFAULT_RETRY_ON_STATUS_CODES,
_adjust_range_header,
_httpx_follow_relative_redirects_with_backoff,
_httpx_follow_hub_redirects_with_backoff,
http_stream_backoff,
)
from .utils._runtime import is_xet_available
Expand Down Expand Up @@ -1618,7 +1618,7 @@ def get_hf_file_metadata(
hf_headers["Accept-Encoding"] = "identity" # prevent any compression => we want to know the real size of the file

# Retrieve metadata
response = _httpx_follow_relative_redirects_with_backoff(
response = _httpx_follow_hub_redirects_with_backoff(
method="HEAD", url=url, headers=hf_headers, timeout=timeout, retry_on_errors=retry_on_errors
)
hf_raise_for_status(response)
Expand Down Expand Up @@ -1740,9 +1740,10 @@ def _get_metadata_or_catch_error(
commit_hash = metadata.commit_hash
if commit_hash is None:
raise FileMetadataError(
"Distant resource does not seem to be on huggingface.co. It is possible that a configuration issue"
" prevents you from downloading resources from https://huggingface.co. Please check your firewall"
" and proxy settings and make sure your SSL certificates are updated."
f"Response from {url} is missing the '{constants.HUGGINGFACE_HEADER_X_REPO_COMMIT}' header, so it"
" does not seem to be served by a Hugging Face Hub endpoint. If HF_ENDPOINT is set, check that it"
" points to a Hub-compatible endpoint. Otherwise, check your firewall and proxy settings and make"
" sure your SSL certificates are updated."
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
)

# Etag must exist
Expand Down Expand Up @@ -1903,6 +1904,11 @@ def _raise_on_head_call_error(head_call_error: Exception, force_download: bool,
# Repo not found or gated => let's raise the actual error
# Unauthorized => likely a token issue => let's raise the actual error
raise head_call_error
elif isinstance(head_call_error, FileMetadataError):
# The call succeeded but the response lacked the metadata we need => a configuration issue, not connectivity.
raise LocalEntryNotFoundError(
f"{head_call_error} We also cannot find the requested files in the local cache."
) from head_call_error
else:
# Otherwise: most likely a connection issue or Hub downtime => let's warn the user
raise LocalEntryNotFoundError(
Expand Down
4 changes: 2 additions & 2 deletions src/huggingface_hub/hf_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@
from .utils import tqdm as hf_tqdm
from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab
from .utils._deprecation import _deprecate_arguments, _deprecate_method
from .utils._http import _httpx_follow_relative_redirects_with_backoff
from .utils._http import _httpx_follow_hub_redirects_with_backoff
from .utils._runtime import is_xet_available
from .utils._typing import CallableT
from .utils._verification import collect_local_files, resolve_local_root, verify_maps
Expand Down Expand Up @@ -14735,7 +14735,7 @@ def get_bucket_file_metadata(
42000
```
"""
response = _httpx_follow_relative_redirects_with_backoff(
response = _httpx_follow_hub_redirects_with_backoff(
"HEAD",
f"{self.endpoint}/buckets/{bucket_id}/resolve/{quote(remote_path, safe='')}",
headers=self._build_hf_headers(token=token),
Expand Down
43 changes: 28 additions & 15 deletions src/huggingface_hub/utils/_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from dataclasses import dataclass
from shlex import quote
from typing import Any, TypeVar
from urllib.parse import urlparse
from urllib.parse import urljoin, urlparse

import httpx

Expand Down Expand Up @@ -691,14 +691,21 @@ def http_stream_backoff(
)


def _httpx_follow_relative_redirects_with_backoff(
_MAX_HUB_REDIRECTS = 10


def _httpx_follow_hub_redirects_with_backoff(
method: HTTP_METHOD_T, url: str, *, retry_on_errors: bool = False, **httpx_kwargs
) -> httpx.Response:
"""Perform an HTTP request with backoff and follow relative redirects only.
"""Perform an HTTP request with backoff, following redirects that stay on the Hub.

Used to fetch HEAD /resolve on repo or bucket files.

This is useful to follow a redirection to a renamed repository without following redirection to a CDN.
A redirect is followed when it targets the host of the current request (e.g. a renamed repository, or the
`/api/resolve-cache/` route) or another known Hub host (e.g. an `HF_ENDPOINT` mirror redirecting to
`huggingface.co`). Redirects to any other host are *not* followed: those point to a CDN or storage bucket, the
file metadata is already carried by the redirect response itself, and the authorization header must not be
forwarded off the Hub.

A backoff mechanism retries the HTTP call on errors (429, 5xx, timeout, network errors).

Expand All @@ -718,7 +725,7 @@ def _httpx_follow_relative_redirects_with_backoff(
{} if retry_on_errors else {"retry_on_exceptions": (), "retry_on_status_codes": ()}
)

while True:
for _ in range(_MAX_HUB_REDIRECTS):
response = http_backoff(
method=method,
url=url,
Expand All @@ -728,18 +735,24 @@ def _httpx_follow_relative_redirects_with_backoff(
)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
hf_raise_for_status(response)

# Check if response is a relative redirect
if 300 <= response.status_code <= 399:
parsed_target = urlparse(response.headers["Location"])
if parsed_target.netloc == "":
# Relative redirect -> update URL and retry
url = urlparse(url)._replace(path=parsed_target.path).geturl()
continue
if not response.has_redirect_location:
return response

target = urljoin(url, response.headers["Location"])

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL (again?) urljoin behavior is so confusing 😵

https://stackoverflow.com/questions/10893374/python-confusions-with-urljoin

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lol Claude sent me to the same stackoverflow page because i didn't understand much in urljoin behavior

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

well 14 years ago and still top 1 google result so we are not the only ones^^

if not _is_same_or_hub_host(url, target):
# Redirect to a CDN or storage host (signed URL): the file metadata is carried by this very response,
# and the authorization header must not be forwarded off the Hub => stop here.
return response

url = target

raise httpx.TooManyRedirects(f"Exceeded maximum of {_MAX_HUB_REDIRECTS} redirects while resolving '{url}'.")

# Break if no relative redirect
break

return response
def _is_same_or_hub_host(url: str, target: str) -> bool:
"""Whether `target` is served by the same host as `url`, or by a known Hub host."""
target_host = (urlparse(target).hostname or "").lower()
return target_host == (urlparse(url).hostname or "").lower() or target_host in constants.HF_URL_HOSTS
Comment thread
hanouticelina marked this conversation as resolved.
Resolved


def fix_hf_endpoint_in_url(url: str, endpoint: str | None) -> str:
Expand Down
Loading