Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
160 commits
Select commit Hold shift + click to select a range
5ec8e27
Post-release: bump version to 1.30.0.dev0 (#4763)
huggingface-hub-bot[bot] Aug 27, 2026
ae20a50
[CI] Enable enforcement in close-unscoped-community-prs workflow (#4766)
hanouticelina Aug 27, 2026
97655e8
Align `hf jobs scheduled ls` filtering with `hf jobs ls` (`--status`/…
moon-bot-app[bot] Aug 28, 2026
1de4817
[Download] Follow redirects between Hub hosts when resolving files (#…
hanouticelina Aug 28, 2026
4cc16bc
[Download] A `ResolvedRevision` only pins the repo it was resolved fo…
Wauplin Aug 28, 2026
94cb25d
[CI] Fix torch_1.11 case-arm label to match renamed matrix entry (#4776)
MohammedAlkindi Aug 31, 2026
6a2eb09
[Sandbox] Document experimental status (#4783)
Wauplin Sep 1, 2026
f0a5c6d
[Inference Providers] Do not require the conversational tag for chat-…
hanouticelina Sep 1, 2026
176d4f3
[CI] Disable auto close (#4800)
Wauplin Sep 2, 2026
b84090b
[Release] Consolidate the HF CLI skill sync into the release workflow…
hanouticelina Sep 3, 2026
c804cdd
Post-release: bump version to 1.31.0.dev0 (#4806)
huggingface-hub-bot[bot] Sep 3, 2026
15bb844
Bump the actions group across 1 directory with 5 updates (#4798)
dependabot[bot] Sep 4, 2026
deb9777
[Docs] Clarify Jobs argument forwarding and align examples (#4809)
davanstrien Sep 4, 2026
634ad9b
[httpx migration] Expose `httpx` as a `huggingface_hub` submodule (#4…
Wauplin Sep 7, 2026
f1bb85f
[hf_api] Make CommitInfo copy-able and pickle-able (#4822)
Wauplin Sep 7, 2026
21e687e
[Download] Prevent cached file copies during local-dir dry runs (#4817)
wakamex Sep 7, 2026
6c8c56b
Fix: keep single-quoted values verbatim in load_dotenv (#4827)
AnishPatel526 Sep 8, 2026
a9b9ac6
[CLI] Fix 'hf buckets rm --recursive' deleting lexical siblings of th…
Wauplin Sep 8, 2026
01fe4bc
[Sandbox] Support custom Job labels (#4824)
Wauplin Sep 8, 2026
204b717
[CLI] Don't let `hf update` self-upgrade a pip install on Windows (#4…
Wauplin Sep 8, 2026
aa3c2fd
[HfFileSystem] Reject unsafe paths before local downloads (#4828)
Wauplin Sep 8, 2026
9d7bbfe
[Download] Share retry handling for stream entry and body failures (#…
Wauplin Sep 8, 2026
8934ec0
[Download] Tolerate missing HEAD Content-Length (#4805)
Wauplin Sep 8, 2026
b9c14d9
[Download] Write cache ref file atomically in `snapshot_download` (#4…
Wauplin Sep 8, 2026
063b37b
[CLI] Raise explicit error for shell-script extensions on Windows (#4…
hanouticelina Sep 9, 2026
6cae778
[Jobs] Add network groups to hf jobs run (#4833)
co42 Sep 9, 2026
22fe960
Fix dotenv parser truncating unquoted values containing '#' (#4842)
BenYang12 Sep 9, 2026
c4f076c
(LFS)Fix SliceFileObj.__iter__ yielding only the first 4MB chunk (#4844)
BenYang12 Sep 9, 2026
f148666
[CLI] Fix truncated command descriptions in the CLI reference (#4849)
hanouticelina Sep 9, 2026
ff4292b
Post-release: bump version to 1.32.0.dev0 (#4856)
huggingface-hub-bot[bot] Sep 10, 2026
a2af2e0
docs(jobs): scope network groups to the namespace and resource group …
co42 Sep 10, 2026
0790f7b
[Community] Fix docstring for DiscussionComment.last_edited_by (#4864)
Saniyagupte Sep 10, 2026
bf85073
fix(sandbox): preserve terminal command results (#4851)
evalstate Sep 10, 2026
819cd77
[CLI] Read a UV script's `[tool.hf-jobs]` launch config (opus-generat…
Wauplin Sep 10, 2026
b853ec6
Add `--container-registry-{username,password}` (#4854)
alvarobartt Sep 10, 2026
e203a7f
fix(serialization): save_torch_state_dict cleanup regex silently matc…
g0rdonL Sep 10, 2026
3493edb
Bump the actions group with 4 updates (#4848)
dependabot[bot] Sep 10, 2026
4110cac
Key the doc build concurrency group on the pull request number (#4876)
albertvillanova Sep 11, 2026
edb2c83
[Buckets] Fix file metadata size on redirects (#4874)
davanstrien Sep 11, 2026
5694eda
[RepoCard] Fix AttributeError when model-index has source: null (#4870)
Saniyagupte Sep 11, 2026
b2678ec
[Spaces] Fix AttributeError in SpaceRuntime when hardware is null (#4…
Saniyagupte Sep 11, 2026
8a408cd
[CI] Make CI less flaky and stop 6h hung jobs (#4830)
Wauplin Sep 11, 2026
dd6919c
[Serialization] Harden torch checkpoint deserialization (#4877)
Wauplin Sep 11, 2026
9588035
feat(jobs): expose the network group on JobInfo (#4862)
co42 Sep 11, 2026
129bbb5
[Http] Fix race in get_session() creating multiple clients (#4868)
hanouticelina Sep 11, 2026
98e2db8
[CLI] Make --format human override agent auto-detection (#4878)
hanouticelina Sep 14, 2026
c5ff040
[Serialization] Sync `_get_dtype_size` with safetensors (#4902)
Wauplin Sep 14, 2026
f1e70a2
[Safetensors] Recognize proposed U3 metadata (#4898)
7dsolv Sep 14, 2026
fb21643
[Download] Fix silently disabled tree cache on long Windows paths (#4…
askalf Sep 14, 2026
490f06e
docs: make the sandbox security contract match the implementation (#4…
Wauplin Sep 14, 2026
1fb5b3b
feat(sandbox): use each pooled sandbox's own capability token (#4832)
Wauplin Sep 14, 2026
ef2662f
fix(sandbox): validate a pool host before sending it a credential (#4…
Wauplin Sep 14, 2026
40efc5e
fix(sandbox): bind the pool cache to the context that wrote it (#4838)
Wauplin Sep 14, 2026
0613ead
fix(sandbox): bound what a transfer or a command's output costs the c…
Wauplin Sep 14, 2026
c7e60d8
fix(sandbox): decide host teardown per host, and report it honestly (…
Wauplin Sep 14, 2026
ec61993
fix(sandbox): address background processes by their server-assigned i…
Wauplin Sep 14, 2026
0219ad5
[sandbox audit] Pin the sandbox server binary by digest and verify it…
Wauplin Sep 14, 2026
1173050
[sandbox audit] Keep secret values out of argv in the CLI and fix the…
Wauplin Sep 14, 2026
48cdc03
Fix doubled colon in snapshot download transfer bar description (#4890)
sandeepb2013 Sep 14, 2026
a3b62f6
[CLI] Add `hf --skills` as an alias for `hf skills preview` (#4906)
Wauplin Sep 14, 2026
bd9ece0
[CLI] Support file URIs in hf cache rm (#4847)
hanouticelina Sep 14, 2026
0a57df9
[Core] Restrict repo_id validation regex to ASCII word characters (#4…
bodapatisaikrishna Sep 15, 2026
af6b41e
[Core] Reject path traversal via embedded ".." segments in path_in_re…
bodapatisaikrishna Sep 15, 2026
2075fd6
[Cache] Support kernel repos in `hf cache` commands (#4905)
hanouticelina Sep 15, 2026
5a9cdda
[Core] Speed up package imports (#4914)
Wauplin Sep 15, 2026
6e3d21f
[Cache] Add cross-repo shared blob store (#4498)
hanouticelina Sep 17, 2026
f94ea55
[Buckets] Send mtime when copying files (#4920)
Wauplin Sep 17, 2026
d7b62c7
[CLI] Fix card data serialization (#4882)
davanstrien Sep 17, 2026
79dd41e
[Docs] Fix hf discussions info options that do not exist (#4886)
rishabhsai Sep 17, 2026
b064a14
[Download] Send X-HF-Download-Counter header on download calls (#4613)
Wauplin Sep 17, 2026
e9c55ce
Post-release: bump version to 1.33.0.dev0 (#4926)
huggingface-hub-bot[bot] Sep 17, 2026
618250b
[Download] Publish symlink support only after probing (#4923)
Wauplin Sep 17, 2026
77b6b2b
[Core] Skip invalid eval results entries instead of raising (#4924)
davanstrien Sep 17, 2026
34ae660
[Cache] Use posix paths for scan_cache_dir ref names (#4925)
gokay-ai Sep 18, 2026
0ba990a
[Download] Measure http_get progress from the initial file position (…
Wauplin Sep 18, 2026
cea607a
[Serialization] Fix device type when map_location is None (#4938)
Wauplin Sep 18, 2026
83ba61a
[CLI] Honor --format / --json / -q on jobs run commands (#4936)
Wauplin Sep 18, 2026
8c9546f
[Quality] Replace `re.match` with `re.fullmatch` / `re.search` and ba…
Wauplin Sep 18, 2026
22b5e87
[Docs] Sync provider task table with the PROVIDERS registry (#4930)
rishabhsai Sep 18, 2026
8efc0a9
[HfFileSystem] Make lpath optional in get_file (fsspec>=2026.9.0) (#4…
assafvayner Sep 21, 2026
4c80e7e
Fix `format_timesince` showing "60 seconds ago" / "60 minutes ago" / …
Sreekant13 Sep 21, 2026
55668e2
Fix Windows conftest erroring test subsets on lazy-loaded file_downlo…
dltsum Sep 21, 2026
7ce68b9
[Webhooks] Allow `bucket` as a watched item type (#4958)
davanstrien Sep 21, 2026
7025391
[MCP] Follow nextCursor when listing tools in MCPClient (#4960)
Kayvan-Zahiri Sep 22, 2026
80aef09
Align the `hf` package's python_requires with huggingface_hub (#4959)
sharziki Sep 22, 2026
94cf392
Remove dots from allowed characters in job labels. (#4962)
evalstate Sep 22, 2026
8d4fba6
[CLI] Honor `CLAUDE_CONFIG_DIR` in `hf skills add -g --claude` (#4957)
hanouticelina Sep 22, 2026
aa8742c
[CLI] Always install skills for Claude Code, deprecate --claude (#4967)
hanouticelina Sep 22, 2026
ce69fdb
[Buckets] Raise when a batch reports failed operations (#4954)
hanouticelina Sep 22, 2026
0f88f32
[Download] Extend tmp .incomplete path on Windows when too long (#4978)
Wauplin Sep 23, 2026
7532272
Fix HfFileSystem.get_file with a bare local filename (#4981)
davanstrien Sep 23, 2026
da5e244
Scope GITHUB_TOKEN permissions per job (#4982)
hf-security-analysis[bot] Sep 23, 2026
1e0c305
Send PrivateLink config under privateService in create_inference_endp…
co42 Sep 23, 2026
b904b94
Apply the repo_id length limit to the namespace as well (#4988)
anishmehta24 Sep 24, 2026
ac3ed2f
[Xet] Show Validating progress during shard finalization (#4478)
seanses Sep 24, 2026
fa26f34
[CI] Support major (pre-)releases in release workflow (#4991)
Wauplin Sep 24, 2026
c2e6e0a
Post-release: bump version to 1.34.0.dev0 (#4993)
huggingface-hub-bot[bot] Sep 24, 2026
9f51867
[Release] Prepare huggingface_hub 2.0 (#4994)
Wauplin Sep 24, 2026
89ae51f
unbump
Wauplin Sep 24, 2026
f5b624c
Post-release: bump version to 2.1.0.dev0 (#4999)
huggingface-hub-bot[bot] Sep 24, 2026
32456ab
[Buckets] Make include/exclude patterns case-sensitive on all platfor…
Wauplin Sep 25, 2026
c749dfa
Support secrets on Job-triggered webhooks (#5009)
moon-bot-app[bot] Sep 25, 2026
ee4f802
[HTTP] Fix repo_id in errors for query strings and download URLs (#5008)
Wauplin Sep 25, 2026
79874bc
[Buckets] Fix copy_files treating lexical siblings as an existing des…
AnishPatel526 Sep 25, 2026
ff0ccab
[CI] Bump doc-builder workflows pin to 63ec6f1 (#5010)
Wauplin Sep 25, 2026
34e8baa
[docs] xet caching and concurrency updates (#4883)
cbensimon Sep 25, 2026
62a1f2f
[HfFileSystem] Download Xet files with hf_xet in get_file (#4990)
hanouticelina Sep 25, 2026
e155ad3
[docs] Fix hf.cos typo in repository guide link (#5016)
pratikgx Sep 28, 2026
a076863
[datasets sql] Run the DuckDB CLI subprocess with UTF-8 encoding (#5020)
825pranav Sep 28, 2026
b15915b
[Auth] Store token names as UTF-8 instead of locale encoding (#5021)
Evolian-o Sep 28, 2026
41222c5
Fix scalar tags splitting into single-character list in card front ma…
zeenat28-ui Sep 28, 2026
14ee250
[CLI] Hint to use --global after local 'hf skills add/update' (#5024)
Wauplin Sep 28, 2026
f151320
[Sandbox] Require scoped credentials for pooled sandboxes (#4782)
Wauplin Sep 28, 2026
f64c26e
[Dataclasses] Accept bare typing.List/Dict/Set in @strict validation …
anishmehta24 Sep 29, 2026
7caeebb
Suggest `hf auth login` when `hf auth whoami` fails (#5033)
moon-bot-app[bot] Sep 29, 2026
5583746
[CLI] Align 'hf auth whoami' not-logged-in message with 'hf auth toke…
Wauplin Sep 29, 2026
862f23d
[Sandbox] Pin sbx-server 0.7.0 (protocol 3) (#5028)
Wauplin Sep 29, 2026
1b8b2fa
[CLI] Escape control characters in listings and table output (#5029)
Wauplin Sep 29, 2026
548f7e3
[Jobs] Allow rescheduling scheduled jobs (#5026)
Wauplin Sep 29, 2026
3fd18f2
[Jobs] Manage public and private exposed ports (#5027)
Wauplin Sep 29, 2026
231a2eb
[Jobs] Add rerun and retry attempts (#5025)
Wauplin Sep 29, 2026
d889712
[Tests] Reduce redundant Hub calls in repo settings and move tests (#…
Wauplin Sep 29, 2026
cc2f368
[Utils] Retry the first pagination page (#5035)
Wauplin Sep 29, 2026
1092497
[Spaces] Update docs for cpu-basic gating and dev mode disabling (#5037)
Wauplin Sep 29, 2026
013afa7
[CLI] Raise if a bare --secrets NAME is not set locally in hf jobs (#…
Wauplin Sep 30, 2026
83a714e
[CLI] Update hf upload help text after upload_large_folder removal (#…
davanstrien Sep 30, 2026
881a6c4
[Docs] Fix links in i18n issue template (#5051)
pratikgx Oct 1, 2026
60fdb08
[Docs] Fix duplicated words in docstring and comments (#5052)
Ad1th Oct 1, 2026
b5e3a12
[HfApi] Support resource_group_id in duplicate_repo (#5038)
Wauplin Oct 1, 2026
3bb08e2
[Endpoints] Migrate catalog to the new /api/v1 API (#4928)
Wauplin Oct 1, 2026
4c88ea6
[Repos] Warn when duplicated repo files are still being copied (#5041)
Wauplin Oct 1, 2026
604252e
[HfApi] Add visibility to create_bucket (#5053)
hanouticelina Oct 1, 2026
0b26b50
[Spaces] Add get_zero_gpu_quota and hf spaces zero-gpu-quota (#5040)
Wauplin Oct 1, 2026
ff61343
Post-release: bump version to 2.2.0.dev0 (#5054)
huggingface-hub-bot[bot] Oct 1, 2026
b3fa3a4
[CLI] Rename 'hf spaces zero-gpu-quota' to 'hf spaces zero-gpu quota'…
Wauplin Oct 1, 2026
7da19ff
[CI] Split tests into api / transfer / inference suites (#5042)
Wauplin Oct 1, 2026
7a54109
[Tests] Always clean up staging repos and buckets created by tests (#…
Wauplin Oct 1, 2026
1a76ad2
[CI] Reduce flaky tests: isolate jedi cache, sleep before read-after-…
Wauplin Oct 1, 2026
c31df86
[Bugfix] Fix wrong exception type and unbound variable guard (#5063)
vector15-05 Oct 2, 2026
37ce057
[CLI] Upload existing paths with glob characters as-is in hf upload (…
Ganesh2325 Oct 2, 2026
0ce4427
[Download] Warn when allow/ignore patterns match no files in snapshot…
Ad1th Oct 2, 2026
a625317
[Repocard] Respect configured endpoint when validating cards (#5069)
Wauplin Oct 2, 2026
d711944
[CLI] Read env/secrets files as UTF-8 and strip the BOM (#5066)
hanouticelina Oct 2, 2026
42f9085
[Setup] Install hf-xet on Windows ARM64 (#5074)
angt Oct 5, 2026
d699913
[Docs] Fix HTTP migration and async session references (#5079)
Yuilona Oct 5, 2026
bc9b691
Bump doc-builder workflow pin (#5087)
paulinebm Oct 5, 2026
3b05c63
[Jobs] Ignore 406 from metrics endpoint for finished jobs (#5091)
hanouticelina Oct 6, 2026
e817417
Bump doc-builder pin to 9fc8a41 in build_documentation.yaml (#5093)
paulinebm Oct 6, 2026
9071b17
[Jobs] Fix no-follow logs hanging while the job keeps printing (#5092)
hanouticelina Oct 6, 2026
68becf4
[CommitScheduler] Fix race that skips the first scheduled push (#5086)
hanouticelina Oct 6, 2026
ff4c989
[CLI] Print [] from hf jobs stats --json when no jobs are running (#5…
hanouticelina Oct 6, 2026
83ce8b0
[Download] Consider port when following /resolve redirects (#4942)
assafvayner Oct 6, 2026
a5277cf
[CLI] Snapshot by default in hf jobs stats, add -f/--follow (#5090)
hanouticelina Oct 6, 2026
14a48e7
[HfFileSystem] fix `ls()` on new empty bucket (#5097)
lhoestq Oct 7, 2026
38e0b3c
[HfFileSystem] Replace cached listings instead of appending to them (…
ChihebBENCHEIKH1 Oct 7, 2026
af74167
[Tests] Wait for bucket listings after writes (#5095)
hanouticelina Oct 7, 2026
cf992a9
[HfFileSystem] Make refresh=True clear cached not-found results (#5106)
hanouticelina Oct 7, 2026
bb3a875
[Docs] Fix from_pretrained integration example (#5105)
Ayushdevo Oct 7, 2026
8abbfe5
Fix race in `_get_version` that makes concurrent downloads skip hf_xe…
BramVanroy Oct 7, 2026
3cfa593
Guard upload_pr_documentation workflow to upstream repo only
g0rdonL Oct 8, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(sandbox): bound what a transfer or a command's output costs the c…
…lient (huggingface#4839)

Four unbounded allocations in the caller's process, each proportional to
something the sandbox controls.

**Output was accumulated even when a callback was consuming it.** `run()`
appended every chunk to `stdout_parts`/`stderr_parts` regardless, so a
streaming consumer still paid full memory for the command's entire output,
and a runaway command (a loop printing to stdout) was an unbounded
allocation. New `capture_output=False` hands output to the callbacks and
drops it; the default now raises above a ceiling instead of being
OOM-killed with no explanation, and the error names the flag to use.

An explicit flag rather than "stop capturing when a callback is present":
silently changing what `result.stdout` contains based on another argument is
a footgun, and code that passes a callback *and* reads `result.stdout` today
keeps working.

**A parallel read doubled peak memory.** `_read_ranges` collected every
chunk into a list and the caller then joined it, so a 2 GB file peaked at
about 4 GB. It is a generator now, yielding `(offset, bytes)` as ranges
land: `read()` places each chunk into one preallocated buffer, `download()`
seeks and writes it out, and both drop it immediately. `read()` also refuses
above a ceiling and points at `download()`, which streams to disk.

**A parallel upload read the whole file into memory.** `upload()` did
`f.read()` above the threshold -- 10 GB resident for a 10 GB file. Each
worker now `pread`s its own range off the shared descriptor (`pread`, not
`seek`+`read`, because the workers share the descriptor and a seeking read
would race with its siblings).

**And it left a stale tail.** A ranged write does not truncate, so
overwriting a larger file with a smaller one left the old bytes past the new
end. Each upload's final chunk now sends the intended total size, which the
server truncates to.

Also follows the server's new listing pagination, so `files.list()` still
returns one complete list however the directory is chunked on the wire --
and the fake server now serves `/files/list` at all, which it never did, so
that path had no coverage.

Validation: 88 tests pass, up from 83. New coverage for opting out of
capture, the runaway-output ceiling and its escape hatch, the read ceiling,
and cursor-following in `list()`.

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
  • Loading branch information
Wauplin and claude authored Sep 14, 2026
commit 0613ead586fb5897052db1edf26da49caa2a83b8
2 changes: 1 addition & 1 deletion docs/source/en/concepts/sandbox.md
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ This section is deliberately exhaustive rather than reassuring: if you are decid
- **Host discovery starts from Job labels**, which any Job creator in the namespace can set — so a label match is a claim, not proof. By default (`adopt_hosts="own"`) the client only adopts hosts *this principal started*, per the Jobs API's `initiator`, and additionally checks the image, flavor, command and exposed URL. Setting `adopt_hosts="namespace"` restores cross-user host sharing and, with it, the ability of any Job creator in that namespace to publish a host your client will send a token to — only use it in a namespace whose members you trust.
- **Landlock can degrade silently.** If Landlock is unavailable, or its ruleset cannot be built, the server currently falls back to uid-only isolation and creates the sandbox anyway — without telling the client. Under uid-only isolation, `/tmp`, `/dev/shm`, TCP bind and cross-home filesystem access are *not* denied. The server also accepts Landlock ABI 1, while the ✅ list above needs ABI 4 (TCP bind) and ABI 6 (abstract sockets); production kernels provide ABI 6, but a lower one would silently drop those two guarantees.
- **Residual shared channels**, none of which Landlock or uid isolation closes: unrestricted outbound TCP; loopback access to the control server; **UDP bind is allowed** (Landlock has no UDP coverage); a sibling's `/proc/<pid>/cmdline` and `status` are readable (`environ` is not — that is the part that would leak credentials, and it is denied); `/proc` and `/sys` are readable and `/dev` is broadly readable and writable; kernel IPC and all machine resources are shared.
- **No CPU, disk, FD or total-memory quotas.** Only per-process `RLIMIT_NPROC` and `RLIMIT_AS` are set; cgroup delegation is not available on Jobs. One sandbox can starve its neighbours. The `max_procs`/`max_mem_mb` values are caller-supplied and not clamped server-side.
- **No CPU-share, disk or total-memory *quota*.** Per-process `RLIMIT_NPROC`, `RLIMIT_AS`, `RLIMIT_NOFILE`, `RLIMIT_FSIZE` and `RLIMIT_CPU` are set, and `max_procs`/`max_mem_mb` are clamped server-side — but cgroup delegation is not available on Jobs, so these are per-process ceilings rather than a share of the host. A sandbox running many processes within its limits can still crowd its neighbours.
- **GPU flavors are untested in pool mode.** The client does not prevent one. Use `Sandbox.create` for GPU.

### Lifecycle and operational gaps
Expand Down
165 changes: 142 additions & 23 deletions src/huggingface_hub/_sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,12 @@

DEFAULT_SANDBOXES_PER_HOST = 50

# Ceiling on the output `run()` will accumulate for its result. Without one, a
# runaway command (a loop printing to stdout, say) is an unbounded allocation in
# the caller's process -- and it happened even when a callback was already
# consuming the output. Raising beats being OOM-killed with no explanation.
MAX_CAPTURED_OUTPUT_CHARS = 64 * 1024 * 1024

# Which pool hosts `create()` may adopt from job labels. Labels are set by whoever
# creates a Job, so "any job carrying our pool's labels" is not an identity claim --
# see the `adopt_hosts` argument of [`SandboxPool`].
Expand Down Expand Up @@ -243,17 +249,37 @@ class SandboxFiles:
PARALLEL_THRESHOLD = 2 * 1024 * 1024
PARALLEL_CHUNK_SIZE = 1 * 1024 * 1024
PARALLEL_MAX_WORKERS = 16
# Ceiling on what `read`/`read_text` will materialize in memory. A parallel
# read used to collect every chunk into a list and *then* join it, so a 2 GB
# file peaked at roughly twice its size before the caller saw a byte. Reading
# a file into memory is inherently bounded by the file; this makes the bound
# explicit and points at the streaming alternative instead of dying in an
# allocator.
MAX_READ_BYTES = 512 * 1024 * 1024

def __init__(self, sandbox: "Sandbox") -> None:
self._sandbox = sandbox

def read(self, path: str) -> bytes:
"""Read a file from the sandbox and return its content as bytes."""
"""Read a file from the sandbox and return its content as bytes.

Raises [`SandboxError`] above `MAX_READ_BYTES`; use [`download`] for
anything that large, which streams to disk instead of buffering.
"""
size = self.stat(path).size
if size > self.MAX_READ_BYTES:
raise SandboxError(
f"{path} is {size} bytes, over the {self.MAX_READ_BYTES}-byte limit for reading into "
"memory. Use `files.download(path, local_path)`, which streams to disk."
)
if size > self.PARALLEL_THRESHOLD:
return b"".join(self._read_ranges(path, size))
response = self._sandbox._request("GET", "/files/read", params={"path": path})
return response.content
# Write each range into one preallocated buffer as it arrives, rather
# than collecting every chunk and joining: the join doubled peak memory.
buffer = bytearray(size)
for offset, part in self._read_ranges(path, size):
buffer[offset : offset + len(part)] = part
return bytes(buffer)
return self._read_range(path, 0, size)

def read_text(self, path: str, encoding: str = "utf-8") -> str:
"""Read a file from the sandbox and return its content as a string."""
Expand Down Expand Up @@ -281,8 +307,12 @@ def upload(self, local_path: str | Path, path: str, mode: str | None = None) ->
whatever the path resolves to a moment later.
"""
with open(local_path, "rb") as f:
if os.fstat(f.fileno()).st_size > self.PARALLEL_THRESHOLD:
self._write_ranges(path, f.read(), mode)
size = os.fstat(f.fileno()).st_size
if size > self.PARALLEL_THRESHOLD:
# Each worker reads its own range straight off the descriptor, so
# a large upload no longer means holding the whole file in memory
# (`f.read()` on a 10 GB file was 10 GB of resident bytes).
self._write_ranges_from_file(path, f, size, mode)
else:
self.write(path, f, mode=mode)

Expand All @@ -296,7 +326,10 @@ def download(self, path: str, local_path: str | Path) -> None:
size = self.stat(path).size
with _open_download_target(Path(local_path)) as f:
if size > self.PARALLEL_THRESHOLD:
for part in self._read_ranges(path, size):
# Seek-and-write per range as it completes, so peak memory is a
# few chunks rather than the whole file.
for offset, part in self._read_ranges(path, size):
f.seek(offset)
f.write(part)
return
with self._sandbox._stream("GET", "/files/read", params={"path": path}) as response:
Expand All @@ -317,30 +350,91 @@ def _parallel(self, items: List[Any], fn: Callable[[Any], Any]) -> List[Any]:
with ThreadPoolExecutor(workers) as executor:
return list(executor.map(fn, items))

def _read_ranges(self, path: str, size: int) -> List[bytes]:
def fetch(rng: tuple[int, int]) -> bytes:
def _read_range(self, path: str, offset: int, length: int) -> bytes:
data = bytearray()
with self._sandbox._stream(
"GET", "/files/read", params={"path": path, "offset": offset, "length": length}
) as response:
for chunk in response.iter_bytes(chunk_size=1024 * 1024):
if len(data) + len(chunk) > length:
raise SandboxError("File response exceeded the requested range.")
data.extend(chunk)
if len(data) != length:
raise SandboxError("File changed size during transfer; retry the download.")
return bytes(data)

def _read_ranges(self, path: str, size: int) -> Iterator[tuple[int, bytes]]:
"""Fetch every range in parallel, yielding `(offset, bytes)` as they land.

Yielding rather than returning a list is the point: the caller places each
chunk and drops it, so peak memory is a few chunks instead of the whole
file plus a copy.
"""

def fetch(rng: tuple[int, int]) -> tuple[int, bytes]:
offset, length = rng
response = self._sandbox._request(
"GET", "/files/read", params={"path": path, "offset": offset, "length": length}
)
return response.content
return offset, self._read_range(path, offset, length)

return self._parallel(self._ranges(size), fetch)
ranges = self._ranges(size)
workers = min(self.PARALLEL_MAX_WORKERS, len(ranges))
with ThreadPoolExecutor(workers) as executor:
for start in range(0, len(ranges), workers):
yield from executor.map(fetch, ranges[start : start + workers])

def _write_ranges(self, path: str, data: bytes, mode: str | None) -> None:
def push(rng: tuple[int, int]) -> None:
offset, length = rng
params: dict[str, Any] = {"path": path, "offset": offset}
if mode is not None:
params["mode"] = mode
self._sandbox._request("PUT", "/files/write", params=params, content=data[offset : offset + length])
self._put_range(path, offset, data[offset : offset + length], mode, total=len(data))

self._parallel(self._ranges(len(data)), push)

def _write_ranges_from_file(self, path: str, source: BinaryIO, size: int, mode: str | None) -> None:
"""Upload `source` in parallel ranges, reading each range on demand.

Serialize reads from the shared descriptor; network writes still overlap.
"""
read_lock = threading.Lock()

def push(rng: tuple[int, int]) -> None:
offset, length = rng
with read_lock:
source.seek(offset)
chunk = source.read(length)
if len(chunk) != length:
raise SandboxError("Upload source changed size during transfer.")
self._put_range(path, offset, chunk, mode, total=size)

self._parallel(self._ranges(size), push)

def _put_range(self, path: str, offset: int, chunk: bytes, mode: str | None, *, total: int) -> None:
params: dict[str, Any] = {"path": path, "offset": offset}
if mode is not None:
params["mode"] = mode
# Tell the server the intended final size. A ranged write does not
# truncate, so overwriting a larger file with a smaller one used to leave
# the old bytes past the new end.
if offset + len(chunk) >= total:
params["truncate_to"] = total
self._sandbox._request("PUT", "/files/write", params=params, content=chunk)

def list(self, path: str) -> List[FileEntry]:
"""List a directory in the sandbox."""
response = self._sandbox._request("GET", "/files/list", params={"path": path})
return [FileEntry(**entry) for entry in response.json()["entries"]]
"""List a directory in the sandbox.

The server paginates, so this follows the cursor and returns everything --
a caller sees one list regardless of how the directory is chunked on the
wire.
"""
entries: List[FileEntry] = []
after: str | None = None
while True:
params: dict[str, Any] = {"path": path}
if after is not None:
params["after"] = after
payload = self._sandbox._request("GET", "/files/list", params=params).json()
entries.extend(FileEntry(**entry) for entry in payload["entries"])
after = payload.get("next")
if not after:
return entries

def stat(self, path: str) -> FileEntry:
"""Get metadata of a file or directory in the sandbox."""
Expand Down Expand Up @@ -873,6 +967,7 @@ def run(
on_stdout: Callable[[str], None] | None = ...,
on_stderr: Callable[[str], None] | None = ...,
check: bool = ...,
capture_output: bool = ...,
background: Literal[False] = ...,
) -> SandboxCommandResult: ...

Expand All @@ -899,6 +994,7 @@ def run(
on_stdout: Callable[[str], None] | None = None,
on_stderr: Callable[[str], None] | None = None,
check: bool = True,
capture_output: bool = True,
background: bool = False,
) -> SandboxCommandResult | SandboxProcess:
"""Run a command in the sandbox and wait for it, streaming output live.
Expand Down Expand Up @@ -932,6 +1028,11 @@ def run(
Callback invoked with stderr chunks as they arrive.
check (`bool`, *optional*, defaults to `True`):
If True, raise [`SandboxCommandError`] on non-zero exit.
capture_output (`bool`, *optional*, defaults to `True`):
If True, accumulate stdout/stderr into the returned result. Pass `False`
when you only want `on_stdout`/`on_stderr`: output is then handed to the
callbacks and dropped, so a command producing gigabytes does not have to fit
in memory. `result.stdout`/`result.stderr` are empty in that case.
background (`bool`, *optional*, defaults to `False`):
If True, start the command detached and return a [`SandboxProcess`] right
away instead of waiting for it and returning a [`SandboxCommandResult`].
Expand All @@ -952,17 +1053,35 @@ def run(
if stdin is not None:
payload["stdin"] = stdin

# Output was accumulated unconditionally, even when a callback was
# consuming it -- so a streaming consumer still paid full memory for the
# command's entire output. `capture_output=False` opts out.
stdout_parts: list[str] = []
stderr_parts: list[str] = []
captured = 0
result: SandboxCommandResult | None = None

def keep(parts: list[str], data: str) -> None:
nonlocal captured
if not capture_output:
return
if captured + len(data) > MAX_CAPTURED_OUTPUT_CHARS:
raise SandboxError(
f"command produced more than {MAX_CAPTURED_OUTPUT_CHARS} characters of output. "
"Pass `capture_output=False` with `on_stdout`/`on_stderr` to stream it instead, "
"or redirect it to a file in the sandbox and download that."
)
captured += len(data)
parts.append(data)

with self._stream("POST", "/exec", json=payload) as response:
for event in _iter_events(response):
if event["event"] == "stdout":
stdout_parts.append(event["data"])
keep(stdout_parts, event["data"])
if on_stdout is not None:
on_stdout(event["data"])
elif event["event"] == "stderr":
stderr_parts.append(event["data"])
keep(stderr_parts, event["data"])
if on_stderr is not None:
on_stderr(event["data"])
elif event["event"] == "exit":
Expand Down
1 change: 1 addition & 0 deletions src/huggingface_hub/cli/sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ def write_stderr(data: str) -> None:
timeout=exec_timeout,
on_stdout=write_stdout,
on_stderr=write_stderr,
capture_output=False,
check=False,
)
if result.timed_out:
Expand Down
78 changes: 78 additions & 0 deletions tests/test_sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,18 @@ def do_GET(self) -> None:
self.send_header("Content-Length", "5")
self.end_headers()
self.wfile.write(b"hello")
elif "/files/list" in self.path:
# Paginated, like the real server: `list_pages` lets a test hand out a
# cursor and check the client follows it.
pages = getattr(cls, "list_pages", None)
if pages:
after = ""
if "after=" in self.path:
after = self.path.split("after=")[1].split("&")[0]
page = pages[0] if not after else next((p for p in pages if p.get("_after") == after), pages[-1])
self._json({"entries": page["entries"], "next": page.get("next")})
else:
self._json({"entries": [], "next": None})
elif self.path.endswith("/processes"): # list background processes
self._json(cls.processes)
elif self.path == "/v1/sandboxes":
Expand Down Expand Up @@ -415,6 +427,72 @@ def test_context_manager_closes_when_reattached(self, fake_server: str) -> None:
assert sandbox._server._client.is_closed


class TestResourceBounds:
"""Client-side memory bounds. Each of these was unbounded: the caller's
process grew with whatever the sandbox produced, whether or not it was
already consuming it."""

def test_parallel_upload_without_pread(self, fake_server, tmp_path, monkeypatch):
monkeypatch.delattr(os, "pread", raising=False)
sandbox = _make_sandbox(fake_server)
monkeypatch.setattr(sandbox.files, "PARALLEL_THRESHOLD", 4)
monkeypatch.setattr(sandbox.files, "PARALLEL_CHUNK_SIZE", 4)
source = tmp_path / "upload.bin"
source.write_bytes(b"abcdefghijkl")
sandbox.files.upload(source, "/dest")
assert sum(len(body) for _, body in _FakeServer.writes) == 12

def test_output_is_not_accumulated_when_the_caller_opts_out(self, fake_server: str) -> None:
sandbox = _make_sandbox(fake_server)
chunks: list = []
result = sandbox.run("echo", on_stdout=chunks.append, capture_output=False)

# The callback still sees everything; the result deliberately holds nothing.
assert chunks == ["out1"]
assert result.stdout == ""
assert result.stderr == ""
assert result.exit_code == 0

def test_output_is_accumulated_by_default(self, fake_server: str) -> None:
sandbox = _make_sandbox(fake_server)
assert sandbox.run("echo").stdout == "out1"

def test_runaway_output_raises_instead_of_growing_without_bound(self, fake_server: str, monkeypatch) -> None:
# Lowered so the test need not actually produce 64 MB.
monkeypatch.setattr(sandbox_mod, "MAX_CAPTURED_OUTPUT_CHARS", 2)
sandbox = _make_sandbox(fake_server)
with pytest.raises(SandboxError, match="capture_output=False"):
sandbox.run("echo")
# ...and the escape hatch the error names actually works.
assert sandbox.run("echo", capture_output=False).stdout == ""

@pytest.mark.parametrize("length", [4, 6])
def test_file_ranges_refuse_short_or_oversized_responses(self, fake_server, length):
sandbox = _make_sandbox(fake_server)
# The server returns five bytes: both a stale stat and a response that
# ignores the requested length must fail rather than corrupt the result.
with pytest.raises(SandboxError):
sandbox.files._read_range("/x", 0, length)

def test_read_refuses_a_file_too_large_to_hold_in_memory(self, fake_server: str, monkeypatch) -> None:
monkeypatch.setattr(sandbox_mod.SandboxFiles, "MAX_READ_BYTES", 1)
sandbox = _make_sandbox(fake_server)
with pytest.raises(SandboxError, match="files.download"):
sandbox.files.read("big.bin")

def test_list_follows_the_servers_pagination(self, fake_server: str) -> None:
# The server pages; a caller should still see one list.
sandbox = _make_sandbox(fake_server)
_FakeServer.list_pages = [
{"entries": [{"name": "a", "path": "/a", "type": "file", "size": 1}], "next": "a"},
{"_after": "a", "entries": [{"name": "b", "path": "/b", "type": "file", "size": 1}]},
]
try:
assert [entry.name for entry in sandbox.files.list("/dir")] == ["a", "b"]
finally:
_FakeServer.list_pages = None


class TestSharedSandbox:
"""A shared sandbox routes operations under /v1/sandboxes/<local_id>/ and is
terminated with a DELETE on the host (the host job keeps running)."""
Expand Down