Repository navigation
[Download] Follow redirects between Hub hosts when resolving files #4739
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
9513105
01944d9
21226b6
47e964f
a749ace
5fdf119
889b938
809e4fb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,7 +26,7 @@ | |
| from dataclasses import dataclass | ||
| from shlex import quote | ||
| from typing import Any, TypeVar | ||
| from urllib.parse import urlparse | ||
| from urllib.parse import urljoin, urlparse | ||
|
|
||
| import httpx | ||
|
|
||
|
|
@@ -691,14 +691,18 @@ def http_stream_backoff( | |
| ) | ||
|
|
||
|
|
||
| def _httpx_follow_relative_redirects_with_backoff( | ||
| _MAX_REDIRECTS = 20 # same bound as httpx's default max_redirects | ||
|
|
||
|
|
||
| def _httpx_follow_hub_redirects_with_backoff( | ||
| method: HTTP_METHOD_T, url: str, *, retry_on_errors: bool = False, **httpx_kwargs | ||
| ) -> httpx.Response: | ||
| """Perform an HTTP request with backoff and follow relative redirects only. | ||
| """Perform an HTTP request with backoff, following redirects that stay on the Hub. | ||
|
|
||
| Used to fetch HEAD /resolve on repo or bucket files. | ||
|
|
||
| This is useful to follow a redirection to a renamed repository without following redirection to a CDN. | ||
| Redirects to the same host or another Hub host are followed. Redirects to any other host (CDN, storage | ||
| bucket) are not: the file metadata is on the redirect response itself and the auth header must not leave the Hub. | ||
|
|
||
| A backoff mechanism retries the HTTP call on errors (429, 5xx, timeout, network errors). | ||
|
|
||
|
|
@@ -718,7 +722,7 @@ def _httpx_follow_relative_redirects_with_backoff( | |
| {} if retry_on_errors else {"retry_on_exceptions": (), "retry_on_status_codes": ()} | ||
| ) | ||
|
|
||
| while True: | ||
| for _ in range(_MAX_REDIRECTS): | ||
| response = http_backoff( | ||
| method=method, | ||
| url=url, | ||
|
|
@@ -728,18 +732,24 @@ def _httpx_follow_relative_redirects_with_backoff( | |
| ) | ||
|
cursor[bot] marked this conversation as resolved.
Resolved
|
||
| hf_raise_for_status(response) | ||
|
|
||
| # Check if response is a relative redirect | ||
| if 300 <= response.status_code <= 399: | ||
| parsed_target = urlparse(response.headers["Location"]) | ||
| if parsed_target.netloc == "": | ||
| # Relative redirect -> update URL and retry | ||
| url = urlparse(url)._replace(path=parsed_target.path).geturl() | ||
| continue | ||
| if not response.has_redirect_location: | ||
| return response | ||
|
|
||
| target = urljoin(url, response.headers["Location"]) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. TIL (again?) https://stackoverflow.com/questions/10893374/python-confusions-with-urljoin
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. lol Claude sent me to the same stackoverflow page because i didn't understand much in
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. well 14 years ago and still top 1 google result so we are not the only ones^^ |
||
| if not _is_same_or_hub_host(url, target): | ||
| # Redirect to a CDN or storage host (signed URL): the file metadata is carried by this very response, | ||
| # and the authorization header must not be forwarded off the Hub => stop here. | ||
| return response | ||
|
|
||
| url = target | ||
|
|
||
| raise httpx.TooManyRedirects(f"Exceeded {_MAX_REDIRECTS} redirects while resolving '{url}'.") | ||
|
|
||
| # Break if no relative redirect | ||
| break | ||
|
|
||
| return response | ||
| def _is_same_or_hub_host(url: str, target: str) -> bool: | ||
| """Whether `target` is served by the same host as `url`, or by a known Hub host.""" | ||
| target_host = (urlparse(target).hostname or "").lower() | ||
| return target_host == (urlparse(url).hostname or "").lower() or target_host in constants.HF_URL_HOSTS | ||
|
hanouticelina marked this conversation as resolved.
Resolved
|
||
|
|
||
|
|
||
| def fix_hf_endpoint_in_url(url: str, endpoint: str | None) -> str: | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.