Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[Download] Measure http_get progress from the initial file position
`http_get` writes into a caller-provided file object, but tracked progress with the
file's absolute position. `HfFileSystem.get_file` explicitly supports handing over an
already-positioned file (it saves `initial_pos` and restores it afterwards), so the
caller's own bytes were counted as downloaded bytes:

- the final size check compared `expected_size` against `temp_file.tell()`, raising a
  spurious "Consistency check failed" for any non-zero starting offset;
- the Range-ignored reset did `seek(0) + truncate()`, destroying the caller's data
  instead of rewinding to where this download began.

Both now work off the bytes downloaded by this call.
  • Loading branch information
askalf committed Sep 14, 2026
commit 771ce2d84c6420bc02de8bfc9af53fa9d8db426e
13 changes: 9 additions & 4 deletions src/huggingface_hub/file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,8 @@ def http_get(
url (`str`):
The URL of the file to download.
temp_file (`BinaryIO`):
The file-like object where to save the file.
The file-like object where to save the file. Content is written from the object's current position, so a
caller may hand over a file that already holds data of its own (see [`HfFileSystem.get_file`]).
resume_size (`int`, *optional*):
The number of bytes already downloaded. If set to 0 (default), the whole file is download. If set to a
positive number, the download will resume at the given position.
Expand Down Expand Up @@ -397,7 +398,9 @@ def http_get(
# If we requested a Range but got 200 back, the server ignored our Range header
# (e.g. CloudFront with Accept-Encoding: gzip). Reset file to avoid corruption.
if resume_size > 0 and response.status_code == 200:
temp_file.seek(0)
# Rewind to where this download started, which is not necessarily the start of the file: the caller
# may have handed over an already-positioned file object.
temp_file.seek(temp_file.tell() - resume_size)
temp_file.truncate()
if _tqdm_bar is not None:
# When the progress bar is reused across retries, its counter has already been advanced by `resume_size`
Expand Down Expand Up @@ -477,10 +480,12 @@ def http_get(
_tqdm_bar=progress,
)

if expected_size is not None and expected_size != temp_file.tell():
# Compare against the bytes downloaded here rather than the absolute file position: the two differ whenever the
# caller passed a file object that was not positioned at 0.
if expected_size is not None and expected_size != new_resume_size:
raise OSError(
consistency_error_message.format(
actual_size=temp_file.tell(),
actual_size=new_resume_size,
)
)

Expand Down
81 changes: 79 additions & 2 deletions tests/test_file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -1324,7 +1324,7 @@ def _mock_response(*, status_code=200, headers, iter_bytes):
return r

@staticmethod
def _http_get_with_mocked_responses(responses, **http_get_kwargs):
def _http_get_with_mocked_responses(responses, *, temp_file=None, **http_get_kwargs):
"""Run http_get against a sequence of mock responses, return the BytesIO."""
it = iter(responses)

Expand All @@ -1333,7 +1333,8 @@ def _mock_stream(*args, **kwargs):
yield next(it)

with patch("huggingface_hub.file_download.http_stream_backoff", side_effect=_mock_stream):
temp_file = io.BytesIO()
if temp_file is None:
temp_file = io.BytesIO()
http_get("fake_url", temp_file=temp_file, **http_get_kwargs)
return temp_file

Expand Down Expand Up @@ -1434,6 +1435,82 @@ def test_http_get_falls_back_to_expected_size_when_response_lacks_content_length
assert tracker.total == 100
assert tracker.n == 100

def test_http_get_on_already_positioned_file(self):
"""Test the consistency check counts downloaded bytes, not the absolute file position.

`HfFileSystem.get_file` hands `http_get` a file object that may already hold data of its own
(it records `initial_pos` and restores it afterwards). Measuring progress with `temp_file.tell()`
then counts the caller's bytes as downloaded bytes and the size check fails spuriously.
"""
temp_file = io.BytesIO()
temp_file.write(b"HEADER--") # caller wrote 8 bytes before handing the file over

self._http_get_with_mocked_responses(
[self._mock_response(headers={"Content-Length": "100"}, iter_bytes=iter([b"A" * 100]))],
temp_file=temp_file,
expected_size=100,
)

assert temp_file.getvalue() == b"HEADER--" + b"A" * 100

def test_http_get_on_already_positioned_file_reports_downloaded_size_on_mismatch(self):
"""Test a genuine size mismatch is still reported, with the downloaded size rather than the position."""
temp_file = io.BytesIO()
temp_file.write(b"HEADER--")

with pytest.raises(OSError, match="file should be of size 100 but has size 50"):
self._http_get_with_mocked_responses(
[self._mock_response(headers={"Content-Length": "100"}, iter_bytes=iter([b"A" * 50]))],
temp_file=temp_file,
expected_size=100,
)

def test_http_get_retry_resets_to_initial_position_when_range_ignored(self):
"""Test the Range-ignored reset rewinds to where this download started, not to the start of the file.

Same setup as `test_http_get_retry_resets_file_when_range_ignored`, but on a file object that is
already positioned: `seek(0) + truncate()` would discard the caller's own bytes.
"""

def _iter_content_1() -> Iterable[bytes]:
yield b"A" * 30
raise httpx.TimeoutException("Fake timeout")

temp_file = io.BytesIO()
temp_file.write(b"HEADER--")

self._http_get_with_mocked_responses(
[
self._mock_response(headers={"Content-Length": "100"}, iter_bytes=_iter_content_1()),
# 200, not 206 — Range was ignored, so the partial bytes are re-sent from scratch
self._mock_response(headers={"Content-Length": "100"}, iter_bytes=iter([b"B" * 100])),
],
temp_file=temp_file,
expected_size=100,
)

assert temp_file.getvalue() == b"HEADER--" + b"B" * 100

def test_http_get_with_resume_size_on_already_positioned_file(self):
"""Test an explicit `resume_size` is honoured on a positioned file: only the missing bytes are counted."""
temp_file = io.BytesIO()
temp_file.write(b"HEADER--" + b"A" * 30) # caller's data + 30 bytes already downloaded

self._http_get_with_mocked_responses(
[
self._mock_response(
status_code=206,
headers={"Content-Length": "70", "Content-Range": "bytes 30-99/100"},
iter_bytes=iter([b"B" * 70]),
)
],
temp_file=temp_file,
expected_size=100,
resume_size=30,
)

assert temp_file.getvalue() == b"HEADER--" + b"A" * 30 + b"B" * 70


class TestCreateSymlink:
@pytest.mark.skipif(os.name == "nt", reason="No symlinks on Windows")
Expand Down