diff --git a/docs/source/en/concepts/sandbox.md b/docs/source/en/concepts/sandbox.md index a6553c5ada0..e15793f33ab 100644 --- a/docs/source/en/concepts/sandbox.md +++ b/docs/source/en/concepts/sandbox.md @@ -190,30 +190,34 @@ A pool is deliberately not a local config file. A pool is its set of running hos Having no authoritative local state is great for correctness but costs latency. A cold `hf sandbox create --pool ` (a fresh CLI process) would otherwise have to rediscover everything over the network before it can create a sandbox: `list_jobs` to scan the namespace → `inspect_job` each host to rebuild its URL and nonce → `GET /v1/sandboxes` to see how full each is → finally `POST` to create. Several round-trips of pure overhead, on every call. -A best-effort cache at `$HF_HOME/sandbox/pools/.json` removes that. After any create/warm, a process records the pool config plus, per host, its proxy URL, auth nonce, and last-seen free slots. The next process rebuilds the host transport straight from the file (no HTTP) and goes directly to the `POST`. +A best-effort cache at `$HF_HOME/sandbox/pools//.json` removes that. After any create/warm, a process records the pool config plus, per host, its proxy URL, auth nonce, and last-seen free slots. The next process rebuilds the host transport straight from the file (no HTTP) and goes directly to the `POST`. + +`` is a digest of the endpoint, the credential and the namespace the entry was written for — the credential as a fingerprint; your token itself is never written to the cache. Both the directory name and the file carry it, and a read that doesn't match on all three is a plain cache miss, so another user of the machine, another endpoint, or a `connect(pool, namespace="other-org")` gets the cold path instead of hosts that were cached for something else. Files are created `0600`, directories `0700`. ```mermaid flowchart TD - start["hf sandbox create --pool ID"] --> rc{"cache hit?
$HF_HOME/sandbox/pools/ID.json"} - rc -- "yes (warm)" --> seed["rebuild host transport
from cached URL + nonce
(no HTTP)"] + start["hf sandbox create --pool ID"] --> rc{"cache hit?
pools/<context>/ID.json
(endpoint + credential + namespace)"} + rc -- "yes, written <15 min ago" --> seed["rebuild host transport
from cached URL + nonce
(no HTTP)"] + rc -- "yes, older" --> check["inspect_job: still a running host
of this pool, on that URL?"] + check -- "yes" --> seed + check -- "no" --> slow seed --> post["POST /v1/sandboxes"] post -- "ok" --> done["sandbox ready
(~1 round-trip)"] post -- "host gone / full" --> slow - rc -- "no / corrupt / stale" --> slow["fallback: list_jobs +
inspect_job + GET (the cold path)"] + rc -- "no / corrupt / another context" --> slow["fallback: list_jobs +
inspect_job + GET (the cold path)"] slow --> post2["POST /v1/sandboxes"] --> done2["sandbox ready
+ refresh cache"] ``` What the cache is and isn't trusted for: - **Not authoritative on capacity.** The in-job server stays authoritative, so a stale `live` count only ever costs a wasted request, never a mis-packed host. +- **Not trusted for where a host lives.** A cached URL is only used if it is the HTTPS jobs-proxy URL of the job that same entry names, so a cache file cannot choose the destination your HF bearer and sandbox token are sent to; an entry naming anything else is dropped from the file. Past 15 minutes an entry is no longer credited on its own either: `inspect_job` has to confirm the job is still running, still labelled for this pool, and still on that URL before the transport is built. +- **Not trusted for its shape.** Types and ranges are checked on read, so a hand-edited, truncated or stringly-typed file is a cache miss rather than an error part-way through `create()`. - **Self-healing.** A cached host that is gone is dropped on the first failed request and pruned from the file; the create transparently falls back to label discovery. - **Concurrency-safe.** Writes merge under a file lock (keyed by `job_id`) and commit atomically, so parallel `create` processes don't clobber each other and readers never see a half-written file. - **Disposable.** Delete it and you get a cache miss: the cold path runs and everything still works. It is never shared across machines. -It *is*, however, trusted for one thing that matters: **the URL a host is reached at.** The fast path rebuilds the transport from the cached `base_url` and nonce and then makes an authenticated request to it, so the first thing that happens to a cached entry is that your HF bearer and sandbox token are sent to whatever URL it names. The file is not authenticated, so it is exactly as trustworthy as `$HF_HOME`: - -- Treat `$HF_HOME` as sensitive. Do not share it between users, restore it from an untrusted source, or place it on a world-writable path. -- The cache is currently keyed by pool id alone, so it is not bound to the endpoint, the credential, or the namespace it was written for. Passing an explicit `namespace=` to `SandboxPool.connect()` that differs from the one the cache was written for will still use the cached hosts. Delete `$HF_HOME/sandbox/pools/.json` if you need to be sure a `connect()` re-discovers from labels. +Two things to keep in mind. Treat `$HF_HOME` as sensitive: an entry is still a reusable host URL plus the public nonce its token derives from, and while a file written under a different credential is ignored, one written under *yours* is credited for the 15-minute window above. And because the namespace is part of the key, a pool cached under an explicit `namespace=` is only found again when the same `namespace=` is passed — `hf sandbox create --pool ` wants the same `--namespace` the pool was created with, or it takes the cold path and looks in your own namespace. ## Known limitations diff --git a/src/huggingface_hub/_sandbox.py b/src/huggingface_hub/_sandbox.py index c76a5ad5c7d..3ade04e2ac6 100644 --- a/src/huggingface_hub/_sandbox.py +++ b/src/huggingface_hub/_sandbox.py @@ -15,6 +15,7 @@ import hashlib import hmac import json +import os import threading import time from concurrent.futures import ThreadPoolExecutor @@ -29,6 +30,8 @@ from . import constants from ._sandbox_cache import ( + HOST_TRUST_TTL, + CacheContext, CachedHost, delete_pool_cache, read_pool_cache, @@ -194,6 +197,32 @@ class FileEntry: mode: str = "" +@contextmanager +def _open_download_target(local_path: Path) -> Iterator[BinaryIO]: + """Yield a writer whose bytes only appear at `local_path` once the download completes. + + Writing straight to the destination has three problems: an interrupted transfer leaves a + truncated file at the final name; `open(path, "wb")` follows a symlink sitting there, so + the sandbox's output can be redirected into any file this process can write; and a + predictable temp name is a race another local process can win. + + So the bytes go to a temp file in the destination directory, created `O_EXCL` (nobody + else's file) and `O_NOFOLLOW` (not a symlink) with mode `0600` and an unguessable name, + then `os.replace`d into place. `rename` does not follow symlinks either, so a symlinked + destination is *replaced* rather than written through, and a failure leaves nothing behind. + """ + tmp = local_path.parent / f".{local_path.name}.{token_hex(8)}.part" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0) + fd = os.open(tmp, flags, 0o600) + try: + with os.fdopen(fd, "wb") as f: + yield f + os.replace(tmp, local_path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + class SandboxFiles: """Filesystem operations inside a sandbox, available as [`Sandbox.files`]. @@ -245,24 +274,32 @@ def write(self, path: str, data: str | bytes | BinaryIO, mode: str | None = None self._sandbox._request("PUT", "/files/write", params=params, content=data) def upload(self, local_path: str | Path, path: str, mode: str | None = None) -> None: - """Upload a local file to the sandbox.""" - size = Path(local_path).stat().st_size - if size > self.PARALLEL_THRESHOLD: - self._write_ranges(path, Path(local_path).read_bytes(), mode) - return + """Upload a local file to the sandbox. + + The file is opened once and both sized (`fstat`) and read through that same + descriptor, so what is uploaded is the file that was measured -- rather than + whatever the path resolves to a moment later. + """ with open(local_path, "rb") as f: - self.write(path, f, mode=mode) + if os.fstat(f.fileno()).st_size > self.PARALLEL_THRESHOLD: + self._write_ranges(path, f.read(), mode) + else: + self.write(path, f, mode=mode) def download(self, path: str, local_path: str | Path) -> None: - """Download a file from the sandbox.""" + """Download a file from the sandbox. + + The bytes land in a private temp file next to the destination and are renamed + over it at the end, so an interrupted download leaves no truncated file and a + symlink sitting at `local_path` is replaced rather than followed. + """ size = self.stat(path).size - if size > self.PARALLEL_THRESHOLD: - with open(local_path, "wb") as f: + with _open_download_target(Path(local_path)) as f: + if size > self.PARALLEL_THRESHOLD: for part in self._read_ranges(path, size): f.write(part) - return - with self._sandbox._stream("GET", "/files/read", params={"path": path}) as response: - with open(local_path, "wb") as f: + return + with self._sandbox._stream("GET", "/files/read", params={"path": path}) as response: for chunk in response.iter_bytes(chunk_size=1024 * 1024): f.write(chunk) @@ -429,7 +466,11 @@ def __init__( headers={"X-Sandbox-Token": sandbox_token}, auth=_SandboxAuth(api), limits=httpx.Limits(max_connections=max_connections, max_keepalive_connections=max_connections), - follow_redirects=True, + # Every request on this client carries the HF bearer and a sandbox token, and there + # is no redirect in this protocol -- the Jobs proxy serves the in-job server + # directly. Following one would hand both credentials to a destination that the + # response, rather than the client, chose. + follow_redirects=False, ) @classmethod @@ -1183,6 +1224,9 @@ def __init__( self.name = name if name is not None else f"pool-{token_hex(6)}" self._idle_timeout = idle_timeout self._namespace = namespace + # Security context the local cache is keyed by: entries written by another endpoint, + # credential or namespace are a miss, not a hit (see `CacheContext`). + self._cache_context = _cache_context(self._api, namespace) self._start_timeout = start_timeout self._adopt_hosts = adopt_hosts # Resolved lazily and only when `adopt_hosts="own"` needs it: whoami() is @@ -1251,10 +1295,14 @@ def connect( token (`str`, *optional*): HF token override. """ + api = HfApi(token=token) # Fast path: rebuild the pool from the local best-effort cache, with no HTTP at all. - # The cached hosts are seeded (and verified) lazily on the next create(); if they are + # Only an entry written by this endpoint, credential and namespace is a hit (see + # `CacheContext`), so `namespace=` is honoured rather than being overridden by whatever + # the file says -- a `connect(namespace="org-B")` can no longer be served org-A's hosts. + # The cached hosts are seeded (and validated) lazily on the next create(); if they are # all stale, create() falls back to label discovery exactly like a cold connect would. - cache = read_pool_cache(pool_id) + cache = read_pool_cache(pool_id, _cache_context(api, namespace)) if cache is not None: return cls( image=cache.image, @@ -1263,14 +1311,13 @@ def connect( max_hosts=cache.max_hosts, name=pool_id, idle_timeout=cache.idle_timeout, - namespace=cache.namespace if namespace is None else namespace, + namespace=namespace, adopt_hosts=adopt_hosts, token=token, _connect_mode=True, # attach to existing hosts; never boot during construction ) # Cold path: find a running host via labels and rebuild the config from its job spec. - api = HfApi(token=token) job = _find_pool_host_job(api, pool_id, namespace=namespace, policy=adopt_hosts) env = _host_env(api, job, namespace=namespace) idle_raw = env.get("SBX_IDLE_TIMEOUT") @@ -1373,7 +1420,7 @@ def create( if self._require_live_host and discovered and not self._hosts: # connect()'d to a pool whose hosts are all gone (stale cache + nothing # found via labels): don't resurrect it under the same id, just report it. - delete_pool_cache(self.name) + delete_pool_cache(self.name, self._cache_context) raise SandboxError( f"No running host found for pool '{self.name}'. The pool has stopped " "(all its hosts were killed or idle-timed-out); create a new one." @@ -1467,7 +1514,8 @@ def close(self) -> None: finally: host.close() if self._owns_hosts: - delete_pool_cache(self.name) # the pool's hosts are gone; don't leave a stale entry + # The pool's hosts are gone; don't leave a stale entry. + delete_pool_cache(self.name, self._cache_context) def __enter__(self) -> "SandboxPool": return self @@ -1521,9 +1569,9 @@ def _ensure_warmed_up(self) -> None: def _principal_id(self) -> str | None: """Id of the principal this pool authenticates as, for the `"own"` policy.""" if not self._principal_resolved: - self._principal_resolved = True try: self._principal = self._api.whoami(cache=True).get("id") + self._principal_resolved = self._principal is not None except Exception as e: # Leave it None: `_adoptable` then refuses rather than guessing. logger.warning(f"Could not resolve the current user to validate pool hosts: {e}") @@ -1839,34 +1887,95 @@ def _drop_host(self, host: _SandboxServer) -> None: def _seed_hosts_from_cache(self) -> None: """Adopt the pool's cached hosts without any HTTP (rebuilt from cached URL + nonce). - Best-effort and unverified: each adopted host is confirmed on the first successful - request (and dropped on the first failure, see `_create_one`). Hosts the cache - believes are full are skipped — label discovery re-checks them with fresh counts if - the seeded ones don't satisfy the request, so a stale-full entry never blocks a create. + The first thing that happens to a seeded host is an authenticated `POST`, so what the + file says about *where* it lives is not taken on trust: the URL must be one only this + job can have (`_cached_host_rejection`), and an entry older than `HOST_TRUST_TTL` is + re-checked against the Jobs API first (`_confirm_cached_host`). A fresh entry — the + `pool create` → `create --pool` case the cache exists for — is used as is. + + Liveness stays best-effort: a seeded host is confirmed by its first successful request + and dropped on the first failure (see `_create_one`). Hosts the cache believes are + full are skipped — label discovery re-checks them with fresh counts if the seeded ones + don't satisfy the request, so a stale-full entry never blocks a create. """ - cache = read_pool_cache(self.name) + cache = read_pool_cache(self.name, self._cache_context) if cache is None: return hf_token = _effective_token(self._api) + now = time.time() + for ch in cache.hosts: + with self._lock: + known = {host.job_id for host in self._hosts} + if ch.job_id in known or ch.live >= ch.capacity: + continue + reason = _cached_host_rejection(ch) + if reason is not None: + logger.debug(f"Dropping cached host {ch.job_id} of pool '{self.name}': it {reason}.") + with self._lock: + self._dead_host_ids.add(ch.job_id) # an entry we can never use is litter + continue + if now - ch.updated_at > HOST_TRUST_TTL and not self._confirm_cached_host(ch): + continue + server = _SandboxServer( + job_id=ch.job_id, + owner=ch.owner, + image=self.image, + base_url=ch.base_url, + nonce=ch.nonce, + sandbox_token=_derive_sandbox_token(hf_token, ch.nonce), + api=self._api, + max_connections=min(self.sandboxes_per_host + 8, 256), + capacity=ch.capacity, + ) + server.live = ch.live + server.verified = False + with self._lock: + if any(host.job_id == ch.job_id for host in self._hosts): + server.close() # seeded concurrently by another thread + else: + self._hosts.append(server) + + def _confirm_cached_host(self, ch: CachedHost) -> bool: + """Confirm an aged-out cache entry against the Jobs API before crediting it. + + Freshness is what the no-HTTP path rests on: the entry was written by this same + principal minutes ago, so the job it names is almost certainly still the host it + described. Past `HOST_TRUST_TTL` that argument weakens, and the entry becomes a hint + that has to be checked — the job must still be running, still be labelled for *this* + pool with the nonce the entry derives its token from, still pass the pool's adoption + policy, and still expose exactly the cached URL. One round-trip, and only for entries + the fast path would no longer be honest about. + + A definitive "no" prunes the entry; a transient failure aborts this attempt + without losing the cached host. + """ + if self._adopt_hosts == ADOPT_OWN and self._principal_id() is None: + raise SandboxError("Could not validate cached pool hosts: current user is unavailable; retry later.") + try: + job = self._api.inspect_job(job_id=ch.job_id, namespace=ch.owner) + labels = job.labels or {} + if job.status.stage != "RUNNING": + reason = f"its job is not running (status: {job.status.stage})" + elif labels.get(MODE_LABEL) != MODE_POOL or labels.get(POOL_LABEL) != self.name: + reason = f"its job is not labelled as a host of pool '{self.name}'" + elif labels.get(NONCE_LABEL) != ch.nonce: + reason = "its job carries a different nonce" + elif not self._adoptable(job): + reason = "its job is not one this pool may adopt" + elif (exposed := _find_server_url(job)) != ch.base_url: + reason = f"its job exposes {exposed!r}, not the cached URL" + else: + return True + except httpx.HTTPStatusError as e: + if e.response.status_code != 404: + raise SandboxError(f"Could not confirm cached host {ch.job_id}; retry later.") from e + reason = "its job no longer exists" + except Exception as e: + raise SandboxError(f"Could not confirm cached host {ch.job_id}; retry later.") from e + logger.debug(f"Dropping stale cached host {ch.job_id} of pool '{self.name}': {reason}.") with self._lock: - known = {host.job_id for host in self._hosts} - for ch in cache.hosts: - if ch.job_id in known or ch.live >= ch.capacity: - continue - server = _SandboxServer( - job_id=ch.job_id, - owner=ch.owner, - image=self.image, - base_url=ch.base_url, - nonce=ch.nonce, - sandbox_token=_derive_sandbox_token(hf_token, ch.nonce), - api=self._api, - max_connections=min(self.sandboxes_per_host + 8, 256), - capacity=ch.capacity, - ) - server.live = ch.live - server.verified = False - self._hosts.append(server) + self._dead_host_ids.add(ch.job_id) + return False def _save_cache(self) -> None: """Persist the pool config + current hosts (with their live counts) for next time.""" @@ -1886,12 +1995,12 @@ def _save_cache(self) -> None: dead = set(self._dead_host_ids) save_pool_cache( self.name, + context=self._cache_context, image=self.image, flavor=self.flavor, sandboxes_per_host=self.sandboxes_per_host, max_hosts=self.max_hosts, idle_timeout=_duration_to_secs(self._idle_timeout) if self._idle_timeout is not None else None, - namespace=self._namespace, hosts=hosts, dead_host_ids=dead, ) @@ -1909,25 +2018,50 @@ def _normalize_image(image: str | None) -> str | None: return image -def _server_url_rejection(job: JobInfo) -> str | None: - """Why this Job's exposed URL is not one we should send credentials to. +def _server_host_rejection(url: str, *, job_id: str) -> str | None: + """Why `url` is not a URL we should send a sandbox job's credentials to. - The hostname must be derived from *this* job's id, so a Job cannot name an - arbitrary destination for the credentials that are about to follow. The - domain is not hard-coded, so staging endpoints keep working. + The hostname must be derived from the job id we already hold, so nothing but the + client's own knowledge of which job it is talking to can decide where the credentials + go. The domain is not hard-coded, so staging endpoints keep working. """ - urls = list(job.status.expose_urls or []) if job.status is not None else [] - if len(urls) != 1: - return f"exposes {len(urls)} URL(s), expected exactly one for port {SANDBOX_SERVER_PORT}" - parsed = urlparse(urls[0]) + parsed = urlparse(url) if parsed.scheme != "https": - return f"exposes the server over {parsed.scheme!r}, not https" + return f"uses scheme {parsed.scheme!r}, not https" # `urlparse` lowercases the hostname, so compare case-insensitively. - if not (parsed.hostname or "").startswith(f"{job.id}--{SANDBOX_SERVER_PORT}.".lower()): - return f"exposes {urls[0]!r}, which is not this job's port-{SANDBOX_SERVER_PORT} URL" + if not (parsed.hostname or "").startswith(f"{job_id}--{SANDBOX_SERVER_PORT}.".lower()): + return f"is not job {job_id}'s port-{SANDBOX_SERVER_PORT} URL" return None +def _server_url_rejection(job: JobInfo) -> str | None: + """Why this Job's exposed URL is not one we should send credentials to.""" + urls = list(job.status.expose_urls or []) if job.status is not None else [] + if len(urls) != 1: + return f"exposes {len(urls)} URL(s), expected exactly one for port {SANDBOX_SERVER_PORT}" + reason = _server_host_rejection(urls[0], job_id=job.id) + return f"exposes {urls[0]!r}, which {reason}" if reason is not None else None + + +def _cached_host_rejection(host: CachedHost) -> str | None: + """Why a cached host entry must not be rebuilt into a credentialed transport. + + Same reasoning as `_server_url_rejection`, for a URL that comes off the local disk + instead of the Jobs API. Without this, the cache file gets to choose where the HF bearer + and the derived host token are sent -- `http://127.0.0.1:1234` included -- and it gets to + choose it *before* any API call could have contradicted it. Deriving the expected + hostname from the entry's own `job_id` leaves it able to name only a job this client can + independently identify, which is the same bar a discovered host has to clear. + """ + reason = _server_host_rejection(host.base_url, job_id=host.job_id) + return f"names {host.base_url!r}, which {reason}" if reason is not None else None + + +def _cache_context(api: HfApi, namespace: str | None) -> CacheContext: + """The local pool cache's security context for this endpoint/credential/namespace.""" + return CacheContext.for_credential(endpoint=api.endpoint, token=_effective_token(api), namespace=namespace) + + def _host_rejection( job: JobInfo, *, diff --git a/src/huggingface_hub/_sandbox_cache.py b/src/huggingface_hub/_sandbox_cache.py index 25ddca75be1..c40bf75dea5 100644 --- a/src/huggingface_hub/_sandbox_cache.py +++ b/src/huggingface_hub/_sandbox_cache.py @@ -14,12 +14,15 @@ # limitations under the License. """Best-effort local cache for SandboxPool hosts (host/pool mode).""" +import hashlib import json import os +import re +import tempfile import time from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import List +from typing import Any, List from . import constants from .utils import WeakFileLock, logging @@ -28,12 +31,83 @@ logger = logging.get_logger(__name__) # Bump if the on-disk layout changes incompatibly; older/newer files are ignored on read. -_CACHE_VERSION = 1 +_CACHE_VERSION = 2 # A write should never block a sandbox creation for long: the cache is best-effort, so we # rather skip persisting than wait on a stuck lock. _LOCK_TIMEOUT = 5.0 +# How long a cached host may be turned into a credentialed transport on the strength of the +# file alone. Within it, the `pool create` -> `create --pool` sequence stays at zero extra +# round-trips (the point of the cache); past it, the caller re-checks the host against the +# Jobs API before sending it anything. +HOST_TRUST_TTL = 15 * 60 + +# The cache holds reusable host URLs and the public nonces the host tokens derive from, so +# both the directories and the files are private to the user. +_DIR_MODE = 0o700 + +# Domain separation, so a digest from this cache is not also a digest of the same material +# computed for some other purpose (and so a version bump renames every directory). +_DIGEST_PREFIX = f"hf-sandbox-pool-cache/v{_CACHE_VERSION}" + +# Plausibility bounds for the schema check below. Generous on purpose: this is here to turn a +# wrong-typed or absurd value into a cache miss, not to re-specify the pool's own limits. +_MAX_COUNT = 1_000_000 +_MAX_SECONDS = 10 * 365 * 24 * 3600 +_MAX_TIMESTAMP = 4_000_000_000 # epoch seconds, some time in 2096 +_MAX_STR = 2048 +# Job ids are backend-assigned; the id also feeds a URL and an `inspect_job` call, so keep it +# to characters that cannot change the meaning of either. +_JOB_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,128}$") +# Nonces are minted by `token_hex(16)`; the bounds are loose so that changing that length +# would not quietly turn every cache entry into a miss. +_NONCE_RE = re.compile(r"^[0-9a-fA-F]{16,128}$") + + +def _digest(*parts: str) -> str: + """Short, stable digest of `parts` (which are joined unambiguously).""" + material = "\x00".join((_DIGEST_PREFIX, *parts)) + return hashlib.sha256(material.encode()).hexdigest() + + +@dataclass(frozen=True) +class CacheContext: + """The security context a cache entry belongs to. + + A cached entry is not just data: the fast path rebuilds a host transport from it and + sends it the HF bearer plus the derived host token. So it may only ever be read back by + the same *endpoint*, *credential* and *namespace* that wrote it -- anything else is a + plain cache miss, which is what the docs promise and what keeps a `connect(namespace=...)` + from being served hosts that belong to another namespace. + + `principal` is an opaque, stable id for the credential; the token itself is never stored + nor used as a key (see `for_credential`). + """ + + endpoint: str + principal: str + namespace: str | None + + @classmethod + def for_credential(cls, *, endpoint: str, token: str, namespace: str | None) -> "CacheContext": + """Build the context of a given HF token, identified by its fingerprint. + + The fingerprint is a local computation, which is what keeps a cache hit at zero + round-trips -- resolving the credential to a user id would cost a `whoami` and defeat + the purpose of the cache. It is derived from the token but does not contain it: only + the digest is ever written to disk. + + A rotated or swapped credential therefore reads as a different principal and gets a + cache miss (the cold path, which always works), rather than someone else's hosts. + """ + return cls(endpoint=endpoint, principal=_digest("principal", token)[:32], namespace=namespace) + + @property + def key(self) -> str: + """Digest of the whole context; the name of the cache subdirectory it owns.""" + return _digest("context", self.endpoint, self.principal, self.namespace or "")[:16] + @dataclass class CachedHost: @@ -64,6 +138,10 @@ class PoolCache: max_hosts: int | None idle_timeout: int | None namespace: str | None + # Context the entry was written in, so a file that is moved (or copied from elsewhere) + # into a directory it was not written for is still rejected. See `CacheContext`. + endpoint: str = "" + context: str = "" hosts: List[CachedHost] = field(default_factory=list) version: int = _CACHE_VERSION updated_at: float = 0.0 @@ -73,43 +151,58 @@ def _pools_dir() -> Path: return Path(constants.HF_HOME) / "sandbox" / "pools" -def pool_cache_path(pool_id: str) -> Path: - """Path of the cache file for `pool_id` (no I/O).""" +def _check_pool_id(pool_id: str) -> str: + """Return `pool_id` if it is safe to use as a file name, else raise.""" if any(c in pool_id for c in ("/", "\\", "\x00")) or pool_id in (".", ".."): raise ValueError(f"Invalid pool id: {pool_id!r}") - return _pools_dir() / f"{pool_id}.json" + return pool_id -def read_pool_cache(pool_id: str) -> PoolCache | None: - """Return the cached view of `pool_id`, or `None` if missing/corrupt/incompatible.""" +def pool_cache_path(pool_id: str, context: CacheContext) -> Path: + """Path of the cache file for `pool_id` in `context` (no I/O).""" + return _pools_dir() / context.key / f"{_check_pool_id(pool_id)}.json" + + +def read_pool_cache(pool_id: str, context: CacheContext) -> PoolCache | None: + """Return the cached view of `pool_id` for `context`, or `None`. + + Missing, corrupt, incompatible, mistyped and foreign-context files are all the same thing + here: a cache miss, logged at debug level. The caller's cold path is always correct, so + this never raises -- a bad cache costs latency, never an error mid-`create()`. + """ try: - path = pool_cache_path(pool_id) + path = pool_cache_path(pool_id, context) with path.open("r", encoding="utf-8") as f: data = json.load(f) if data.get("version") != _CACHE_VERSION: return None hosts = [CachedHost(**h) for h in data.pop("hosts", [])] - return PoolCache(**data, hosts=hosts) + cache = PoolCache(**data, hosts=hosts) except FileNotFoundError: return None except Exception as e: logger.debug(f"Ignoring unreadable sandbox pool cache for {pool_id!r}: {e}") return None + reason = _cache_rejection(cache, pool_id=pool_id, context=context) + if reason is not None: + logger.debug(f"Ignoring sandbox pool cache for {pool_id!r}: {reason}.") + return None + return cache def save_pool_cache( pool_id: str, *, + context: CacheContext, image: str, flavor: str, sandboxes_per_host: int, max_hosts: int | None, idle_timeout: int | None, - namespace: str | None, hosts: List[CachedHost], dead_host_ids: set[str] | None = None, ) -> None: - """Merge `hosts` into the cache for `pool_id` (best-effort, never raises). + """Merge `hosts` into the cache for `pool_id` in `context` (best-effort, never raises). Concurrency-safe: under a file lock, the on-disk hosts are read, then `hosts` are upserted by `job_id` and `dead_host_ids` removed, so a process only adds/updates what @@ -118,10 +211,10 @@ def save_pool_cache( """ dead = dead_host_ids or set() try: - path = pool_cache_path(pool_id) - path.parent.mkdir(parents=True, exist_ok=True) + path = pool_cache_path(pool_id, context) + _ensure_private_dir(path.parent) with WeakFileLock(str(path) + ".lock", timeout=_LOCK_TIMEOUT): - existing = read_pool_cache(pool_id) + existing = read_pool_cache(pool_id, context) merged = {h.job_id: h for h in (existing.hosts if existing else [])} for host in hosts: merged[host.job_id] = host @@ -134,7 +227,9 @@ def save_pool_cache( sandboxes_per_host=sandboxes_per_host, max_hosts=max_hosts, idle_timeout=idle_timeout, - namespace=namespace, + namespace=context.namespace, + endpoint=context.endpoint, + context=context.key, hosts=list(merged.values()), updated_at=time.time(), ) @@ -143,17 +238,114 @@ def save_pool_cache( logger.debug(f"Could not write sandbox pool cache for {pool_id!r}: {e}") -def delete_pool_cache(pool_id: str) -> None: - """Remove the cache file for `pool_id` (best-effort, never raises).""" +def delete_pool_cache(pool_id: str, context: CacheContext | None = None) -> None: + """Remove the cache file(s) for `pool_id` (best-effort, never raises). + + Without a `context`, the pool's entry is removed from *every* context: deleting a pool is + the one operation for which over-deleting is the safe direction (the cache is disposable), + and the caller may not know which credential wrote the entry. + """ try: - pool_cache_path(pool_id).unlink(missing_ok=True) + if context is not None: + pool_cache_path(pool_id, context).unlink(missing_ok=True) + return + filename = f"{_check_pool_id(pool_id)}.json" + for directory in _pools_dir().iterdir(): + if directory.is_dir(): + (directory / filename).unlink(missing_ok=True) except Exception as e: logger.debug(f"Could not delete sandbox pool cache for {pool_id}: {e}") +def _ensure_private_dir(path: Path) -> None: + """Create the cache directory tree, readable by this user only. + + `mkdir`'s mode is masked by the umask and the tree may predate this code, so the modes + are also set explicitly, from the leaf up to (and including) the sandbox directory. + """ + path.mkdir(parents=True, exist_ok=True, mode=_DIR_MODE) + root = Path(constants.HF_HOME) / "sandbox" + for directory in (path, *path.parents): + if directory != root and root not in directory.parents: + break # $HF_HOME itself (and anything above it) is not ours to re-mode + os.chmod(directory, _DIR_MODE) + + def _atomic_write(path: Path, cache: PoolCache) -> None: - """Write the cache via a temp file + `os.replace` so readers never see a partial file.""" - tmp = path.parent / f"{path.name}.{os.getpid()}.tmp" - with tmp.open("w", encoding="utf-8") as f: - json.dump(asdict(cache), f, indent=2) - os.replace(tmp, path) + """Write the cache via a temp file + `os.replace` so readers never see a partial file. + + `mkstemp` gives the temp file an unpredictable name and mode `0600` regardless of the + umask (which `os.replace` then carries over to the final file), so no other user can read + -- or pre-create and win a race on -- the host URLs and nonces on their way to disk. + """ + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(asdict(cache), f, indent=2) + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + + +def _is_count(value: Any, *, minimum: int = 0, maximum: int = _MAX_COUNT) -> bool: + """Whether `value` is a plausible count. `bool` is an `int`, but not a count.""" + return isinstance(value, int) and not isinstance(value, bool) and minimum <= value <= maximum + + +def _is_timestamp(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= _MAX_TIMESTAMP + + +def _is_text(value: Any, *, allow_empty: bool = False) -> bool: + return isinstance(value, str) and (allow_empty or len(value) > 0) and len(value) <= _MAX_STR + + +def _cache_rejection(cache: PoolCache, *, pool_id: str, context: CacheContext) -> str | None: + """Why this cache file must not be used, or `None` if it may be. + + Two jobs. First, the entry has to belong to the caller's security context -- same + endpoint, same credential, same namespace -- which is what makes a cross-namespace or + cross-principal hit a miss instead of a confidentiality bug. + + Second, a schema check. `dataclass` does not enforce types, so a well-shaped file with + `"capacity": "50"` parses happily and only blows up much later, on `host.capacity - + host.live`, as an uncaught `TypeError` in the middle of `create()`. Validating here turns + that into the documented cache miss. + """ + if cache.context != context.key: + return "it was written for a different endpoint, credential or namespace" + if cache.endpoint != context.endpoint: + return f"it names endpoint {cache.endpoint!r}, not {context.endpoint!r}" + if cache.namespace != context.namespace: + return f"it names namespace {cache.namespace!r}, not {context.namespace!r}" + if cache.pool_id != pool_id: + return f"it names pool {cache.pool_id!r}, not {pool_id!r}" + + if not _is_text(cache.image) or not _is_text(cache.flavor): + return "its image/flavor are not strings" + if not _is_count(cache.sandboxes_per_host, minimum=1): + return f"its sandboxes_per_host is not a positive count: {cache.sandboxes_per_host!r}" + if cache.max_hosts is not None and not _is_count(cache.max_hosts, minimum=1): + return f"its max_hosts is not a positive count: {cache.max_hosts!r}" + if cache.idle_timeout is not None and not _is_count(cache.idle_timeout, maximum=_MAX_SECONDS): + return f"its idle_timeout is not a duration in seconds: {cache.idle_timeout!r}" + if not _is_timestamp(cache.updated_at): + return f"its updated_at is not a timestamp: {cache.updated_at!r}" + + if not isinstance(cache.hosts, list): + return "its hosts are not a list" + for host in cache.hosts: + if not _JOB_ID_RE.match(host.job_id if isinstance(host.job_id, str) else ""): + return f"host {host.job_id!r} is not a plausible job id" + if not _JOB_ID_RE.match(host.owner if isinstance(host.owner, str) else ""): + return f"host {host.job_id} names a non-namespace owner: {host.owner!r}" + if not _is_text(host.base_url): + return f"host {host.job_id} has no URL" + if not isinstance(host.nonce, str) or not _NONCE_RE.match(host.nonce): + return f"host {host.job_id} has no usable nonce" + if not _is_count(host.capacity) or not _is_count(host.live): + return f"host {host.job_id} has a non-numeric capacity/live count" + if not _is_timestamp(host.updated_at): + return f"host {host.job_id} has no usable timestamp" + return None diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 9dd3e1afc35..898ba1382dd 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -1,5 +1,8 @@ import json +import os +import stat import threading +import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest.mock import MagicMock @@ -14,14 +17,26 @@ NONCE_LABEL, POOL_LABEL, SANDBOX_LABEL, + SANDBOX_SERVER_PORT, Sandbox, SandboxPool, _SandboxServer, ) -from huggingface_hub._sandbox_cache import CachedHost, read_pool_cache, save_pool_cache +from huggingface_hub._sandbox_cache import ( + HOST_TRUST_TTL, + CacheContext, + CachedHost, + read_pool_cache, + save_pool_cache, +) from huggingface_hub.errors import SandboxCommandError, SandboxError +# A job nonce shaped like the real thing (minted by `token_hex(16)`): the public label the +# sandbox token derives from, and a value the pool cache checks the shape of on read. +NONCE = "1e" * 16 + + def _fake_list_jobs(jobs): """Stand-in for `HfApi.list_jobs` that mimics the server-side `status`/`labels` filtering. @@ -60,7 +75,7 @@ def _make_server(base_url: str, job_id: str = "job123", capacity: int = 0) -> _S owner="user", image="python:3.12", base_url=base_url, - nonce="nonce", + nonce=NONCE, sandbox_token="secret", api=api, capacity=capacity, @@ -118,10 +133,18 @@ class _FakeServer(BaseHTTPRequestHandler): last_exec: dict | None = None # body of the most recent /exec call (for assertions) processes: list = [] # background processes started via /processes proc_seq = 0 # monotonic process id source + # Every request that reached this server, as (method, path). A server that must never be + # contacted at all (a URL only a cache file named, a redirect target) is asserted on this. + requests: list = [] + writes: list = [] # (path, body) of every /files/write received + redirect_to = "" # where the /v1/redirect route points (set per-subclass) def log_message(self, *args) -> None: pass + def _record(self) -> None: + type(self).requests.append((self.command, self.path)) + def _ndjson(self, events) -> None: self.send_response(200) self.send_header("Content-Type", "application/x-ndjson") @@ -165,6 +188,7 @@ def _exec(self, body) -> None: ) def do_POST(self) -> None: + self._record() self._expect_token() body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") cls = type(self) @@ -198,7 +222,15 @@ def do_POST(self) -> None: created.append({"id": sid, "token": f"tok-{sid}"}) self._json({"sandboxes": created, "rejected": rejected}) + def do_PUT(self) -> None: + self._record() + self._expect_token() + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + type(self).writes.append((self.path, body)) + self._json({"written": len(body)}) + def do_DELETE(self) -> None: + self._record() self._expect_token() last = self.path.rsplit("/", 1)[-1] if "/processes/" in self.path: # kill a background process @@ -209,9 +241,15 @@ def do_DELETE(self) -> None: self._json({"id": last, "deleted": True}) def do_GET(self) -> None: + self._record() self._expect_token() cls = type(self) - if self.path.startswith("/v1/files/stat") or "/files/stat" in self.path: + if self.path == "/v1/redirect": # a redirect the client must not follow + self.send_response(302) + self.send_header("Location", cls.redirect_to) + self.send_header("Content-Length", "0") + self.end_headers() + elif self.path.startswith("/v1/files/stat") or "/files/stat" in self.path: self._json({"name": "x", "path": "/x", "type": "file", "size": 5}) elif self.path.startswith("/v1/files/read") or "/files/read" in self.path: self.send_response(200) @@ -235,6 +273,9 @@ def fake_server(): _FakeServer.last_exec = None _FakeServer.processes = [] _FakeServer.proc_seq = 0 + _FakeServer.requests = [] + _FakeServer.writes = [] + _FakeServer.redirect_to = "" server = ThreadingHTTPServer(("127.0.0.1", 0), _FakeServer) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -250,6 +291,8 @@ class _Fake(_FakeServer): seq = 0 processes: list = [] proc_seq = 0 + requests: list = [] + writes: list = [] _Fake.capacity = capacity server = ThreadingHTTPServer(("127.0.0.1", 0), _Fake) @@ -549,7 +592,7 @@ def _pool_host_job( job.command = ["/bin/sh", "-c", sandbox_mod._BOOTSTRAP_DOWNLOAD] job.status.stage = "RUNNING" job.status.expose_urls = [f"https://{job_id}--{sandbox_mod.SANDBOX_SERVER_PORT}.hf.jobs"] - job.labels = {SANDBOX_LABEL: "1", MODE_LABEL: MODE_POOL, POOL_LABEL: pool_name, NONCE_LABEL: "nonce"} + job.labels = {SANDBOX_LABEL: "1", MODE_LABEL: MODE_POOL, POOL_LABEL: pool_name, NONCE_LABEL: NONCE} job.environment = {"SBX_CAPACITY": str(capacity)} if env is None else env return job @@ -802,7 +845,39 @@ def test_connect_raises_when_pool_gone(self, monkeypatch) -> None: SandboxPool.connect("pool-dead", token="hf_test") -def _save_cache(pool_id: str, hosts, **overrides) -> None: +def _ctx(namespace: str | None = None, *, token: str = "hf_test", endpoint: str | None = None) -> CacheContext: + """The cache context a pool with this endpoint / credential / namespace reads and writes under.""" + return sandbox_mod._cache_context(sandbox_mod.HfApi(token=token, endpoint=endpoint), namespace) + + +def _cached_host( + job_id: str = "h1", + *, + capacity: int = 4, + live: int = 0, + url: str | None = None, + nonce: str = NONCE, + owner: str = "user", + age: float = 0.0, +) -> CachedHost: + """A cache entry as a real run would have written it, `age` seconds ago. + + The default URL is the one the entry's own job would expose, because that is the only + kind the client will rebuild a transport from -- a fixture naming anything else would be + testing a code path that no longer exists. + """ + return CachedHost( + job_id=job_id, + owner=owner, + base_url=url if url is not None else f"https://{job_id}--{SANDBOX_SERVER_PORT}.hf.jobs", + nonce=nonce, + capacity=capacity, + live=live, + updated_at=time.time() - age, + ) + + +def _save_cache(pool_id: str, hosts, *, context: CacheContext | None = None, **overrides) -> None: """Write a pool cache with sane defaults, overriding config fields as needed.""" config = { "image": "python:3.12", @@ -810,88 +885,130 @@ def _save_cache(pool_id: str, hosts, **overrides) -> None: "sandboxes_per_host": 4, "max_hosts": None, "idle_timeout": 600, - "namespace": None, **overrides, } - save_pool_cache(pool_id, hosts=hosts, **config) + save_pool_cache(pool_id, context=context if context is not None else _ctx(), hosts=hosts, **config) + + +@pytest.fixture() +def loopback_cache_hosts(monkeypatch): + """Let a cached host point at a local fake server. + + A cached URL is only admitted if it is the HTTPS jobs-proxy URL of the entry's own job, + which a loopback test server can never be (that check is exercised directly in + `TestCachedHostAdmission`). The tests using this fixture are about what the cache does + once a host *is* seeded, so they opt out of that one check and nothing else. + """ + monkeypatch.setattr(sandbox_mod, "_cached_host_rejection", lambda host: None) class TestPoolCacheFile: - """Unit tests for the on-disk cache layout (`$HF_HOME/sandbox/pools/.json`).""" + """Unit tests for the on-disk cache layout (`$HF_HOME/sandbox/pools//.json`).""" def test_round_trip(self) -> None: - _save_cache("p", [CachedHost("h1", "user", "http://h1", "n1", 4, 1)], namespace="ns") - cache = read_pool_cache("p") + _save_cache("p", [_cached_host("h1", live=1)], context=_ctx("ns")) + cache = read_pool_cache("p", _ctx("ns")) assert cache is not None assert (cache.image, cache.sandboxes_per_host, cache.namespace) == ("python:3.12", 4, "ns") assert cache.hosts[0].job_id == "h1" and cache.hosts[0].live == 1 def test_missing_returns_none(self) -> None: - assert read_pool_cache("does-not-exist") is None + assert read_pool_cache("does-not-exist", _ctx()) is None def test_path_rejects_traversal(self) -> None: for bad in ("../evil", "a/b", "..", "x\x00y"): with pytest.raises(ValueError): - cache_mod.pool_cache_path(bad) - # read/save stay best-effort (no raise) even for an invalid id. - assert read_pool_cache("../evil") is None - _save_cache("../evil", [CachedHost("h1", "user", "u", "n", 4, 0)]) # no raise + cache_mod.pool_cache_path(bad, _ctx()) + # read/save/delete stay best-effort (no raise) even for an invalid id. + assert read_pool_cache("../evil", _ctx()) is None + _save_cache("../evil", [_cached_host()]) # no raise + cache_mod.delete_pool_cache("../evil") # no raise def test_corrupt_returns_none(self) -> None: - path = cache_mod.pool_cache_path("bad") + path = cache_mod.pool_cache_path("bad", _ctx()) path.parent.mkdir(parents=True, exist_ok=True) path.write_text("{ not valid json") - assert read_pool_cache("bad") is None # tolerated as a cache miss + assert read_pool_cache("bad", _ctx()) is None # tolerated as a cache miss def test_version_mismatch_returns_none(self, monkeypatch) -> None: - _save_cache("p", [CachedHost("h1", "user", "http://h1", "n1", 4, 0)]) + _save_cache("p", [_cached_host()]) monkeypatch.setattr(cache_mod, "_CACHE_VERSION", 999) - assert read_pool_cache("p") is None + assert read_pool_cache("p", _ctx()) is None def test_merge_upserts_and_prunes(self) -> None: - _save_cache( - "p", [CachedHost("h1", "user", "http://h1", "n1", 4, 0), CachedHost("h2", "user", "u2", "n2", 4, 0)] - ) + _save_cache("p", [_cached_host("h1"), _cached_host("h2")]) # A second writer updates h2's live count and reports h1 as dead. - _save_cache("p", [CachedHost("h2", "user", "u2", "n2", 4, 3)], dead_host_ids={"h1"}) - cache = read_pool_cache("p") + _save_cache("p", [_cached_host("h2", live=3)], dead_host_ids={"h1"}) + cache = read_pool_cache("p", _ctx()) assert cache is not None assert {h.job_id for h in cache.hosts} == {"h2"} # h1 pruned, h2 kept assert cache.hosts[0].live == 3 # updated value won def test_delete(self) -> None: - _save_cache("p", [CachedHost("h1", "user", "http://h1", "n1", 4, 0)]) + _save_cache("p", [_cached_host()]) + cache_mod.delete_pool_cache("p", _ctx()) + assert read_pool_cache("p", _ctx()) is None + + def test_delete_without_a_context_clears_every_context(self) -> None: + # `hf sandbox pool delete` knows the pool id, not which credential cached it. Over- + # deleting a disposable cache is the safe direction, under-deleting leaves a stale entry. + _save_cache("p", [_cached_host()], context=_ctx("org-a")) + _save_cache("p", [_cached_host()], context=_ctx("org-b")) cache_mod.delete_pool_cache("p") - assert read_pool_cache("p") is None + assert read_pool_cache("p", _ctx("org-a")) is None + assert read_pool_cache("p", _ctx("org-b")) is None + + @pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") + def test_the_cache_is_private_to_this_user(self) -> None: + # It holds reusable host URLs and the public nonces the host tokens derive from. + _save_cache("p", [_cached_host()]) + path = cache_mod.pool_cache_path("p", _ctx()) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert stat.S_IMODE(path.parent.stat().st_mode) == 0o700 # the context dir + assert stat.S_IMODE(path.parent.parent.stat().st_mode) == 0o700 # sandbox/pools + # No temp file left behind (and none with a name another process could have guessed). + assert {p.name for p in path.parent.iterdir()} <= {"p.json", "p.json.lock"} + + @pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") + def test_a_lax_pre_existing_directory_is_tightened(self) -> None: + directory = cache_mod._pools_dir() + directory.mkdir(parents=True, exist_ok=True) + directory.chmod(0o777) # e.g. written by an older version, under a lax umask + _save_cache("p", [_cached_host()]) + assert stat.S_IMODE(directory.stat().st_mode) == 0o700 class TestPoolCacheIntegration: """`SandboxPool.connect` + `create` use the cache to skip list_jobs/inspect_job when warm.""" - def test_warm_cache_packs_without_listing_jobs(self, fake_server: str, monkeypatch) -> None: + def test_warm_cache_packs_without_listing_jobs(self, fake_server: str, monkeypatch, loopback_cache_hosts) -> None: # An earlier run left a warm host in the cache; a fresh pool must reach it with no HTTP # other than the create POST itself. - _save_cache("pool-cached", [CachedHost("host-cached", "user", fake_server, "nonce", 4, 0)]) + _save_cache("pool-cached", [_cached_host("host-cached", url=fake_server)]) monkeypatch.setattr(sandbox_mod, "_derive_sandbox_token", lambda *a: "secret") pool = SandboxPool.connect("pool-cached", token="hf_test") pool._api.list_jobs = MagicMock(side_effect=AssertionError("must not list jobs")) + pool._api.inspect_job = MagicMock(side_effect=AssertionError("must not inspect the host job")) monkeypatch.setattr(pool, "_boot_host", lambda: pytest.fail("must not boot a host")) box = pool.create() assert isinstance(box, Sandbox) and box.host_id == "host-cached" pool._api.list_jobs.assert_not_called() - cache = read_pool_cache("pool-cached") + pool._api.inspect_job.assert_not_called() # a fresh entry costs no extra round-trip + assert _FakeServer.requests == [("POST", "/v1/sandboxes")] # ...and exactly one request + cache = read_pool_cache("pool-cached", _ctx()) assert cache is not None and cache.hosts[0].live == 1 # live count persisted back - def test_stale_host_falls_back_to_discovery_and_prunes(self, fake_server: str, monkeypatch) -> None: + def test_stale_host_falls_back_to_discovery_and_prunes( + self, fake_server: str, monkeypatch, loopback_cache_hosts + ) -> None: # The cached host is dead (refused connection); discovery finds a live one via labels. - _save_cache("pool-stale", [CachedHost("dead", "user", "http://127.0.0.1:1", "n", 4, 0)]) + _save_cache("pool-stale", [_cached_host("dead", url="http://127.0.0.1:1")]) monkeypatch.setattr(sandbox_mod, "_derive_sandbox_token", lambda *a: "secret") pool = SandboxPool.connect("pool-stale", token="hf_test") live_job = _pool_host_job("live", pool_name="pool-stale") - live_job.labels = {SANDBOX_LABEL: "1", MODE_LABEL: MODE_POOL, POOL_LABEL: "pool-stale", NONCE_LABEL: "n"} pool._api.list_jobs = MagicMock(return_value=[live_job]) monkeypatch.setattr( sandbox_mod, "_connect_host", lambda api, jid, namespace=None: _make_server(fake_server, job_id=jid) @@ -899,13 +1016,13 @@ def test_stale_host_falls_back_to_discovery_and_prunes(self, fake_server: str, m box = pool.create() assert box.host_id == "live" - cache = read_pool_cache("pool-stale") + cache = read_pool_cache("pool-stale", _ctx()) assert cache is not None and [h.job_id for h in cache.hosts] == ["live"] # dead host pruned - def test_stale_cache_does_not_resurrect_dead_pool(self, monkeypatch) -> None: + def test_stale_cache_does_not_resurrect_dead_pool(self, monkeypatch, loopback_cache_hosts) -> None: # connect() trusted a stale cache, but every host is gone and labels find nothing: # create() must refuse to boot a fresh host under the same id, and clear the cache. - _save_cache("pool-ghost", [CachedHost("dead", "user", "http://127.0.0.1:1", "n", 4, 0)]) + _save_cache("pool-ghost", [_cached_host("dead", url="http://127.0.0.1:1")]) monkeypatch.setattr(sandbox_mod, "_derive_sandbox_token", lambda *a: "secret") pool = SandboxPool.connect("pool-ghost", token="hf_test") pool._api.list_jobs = MagicMock(return_value=[]) @@ -913,7 +1030,7 @@ def test_stale_cache_does_not_resurrect_dead_pool(self, monkeypatch) -> None: with pytest.raises(SandboxError, match="No running host found"): pool.create() - assert read_pool_cache("pool-ghost") is None # cleared + assert read_pool_cache("pool-ghost", _ctx()) is None # cleared @pytest.mark.parametrize("exit_code,timed_out", [(0, False), (3, False), (0, True), (None, True)]) @@ -975,3 +1092,297 @@ def test_terminal_stream_ignores_events_after_exit(terminal_stream): callback.assert_not_called() assert response.is_closed and stream.closed handler.assert_called_once() + + +class TestPoolCacheBinding: + """The cache is a credential delivery instruction, not just data: the fast path rebuilds a + host transport from a file and sends it the HF bearer plus the derived host token. So an + entry is only ever readable by the endpoint, credential and namespace that wrote it, and + only if it still says what it said when it was written.""" + + def test_delete_treats_pool_names_literally(self) -> None: + for name in ("p*", "p1", "p[1]"): + _save_cache(name, [_cached_host()]) + sandbox_mod.delete_pool_cache("p*") + assert read_pool_cache("p*", _ctx()) is None + assert read_pool_cache("p1", _ctx()) is not None + assert read_pool_cache("p[1]", _ctx()) is not None + + def test_another_credential_gets_a_miss(self) -> None: + _save_cache("p", [_cached_host()], context=_ctx(token="hf_someone_else")) + assert read_pool_cache("p", _ctx()) is None + + def test_another_namespace_gets_a_miss(self) -> None: + _save_cache("p", [_cached_host()], context=_ctx("org-a")) + assert read_pool_cache("p", _ctx("org-b")) is None + assert read_pool_cache("p", _ctx()) is None # nor does "no namespace" match one + assert read_pool_cache("p", _ctx("org-a")) is not None + + def test_another_endpoint_gets_a_miss(self) -> None: + _save_cache("p", [_cached_host()], context=_ctx(endpoint="https://hub.staging.example")) + assert read_pool_cache("p", _ctx()) is None + + def test_an_entry_moved_into_our_directory_is_still_a_miss(self) -> None: + # The context is in the payload as well as the path, so copying a file (from a + # colleague's $HF_HOME, say) into the right-looking directory does not make it ours. + _save_cache("p", [_cached_host()], context=_ctx("org-a")) + source = cache_mod.pool_cache_path("p", _ctx("org-a")) + target = cache_mod.pool_cache_path("p", _ctx()) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(source.read_text()) + assert read_pool_cache("p", _ctx()) is None + + def test_a_renamed_entry_is_a_miss(self) -> None: + _save_cache("mine", [_cached_host()]) + source = cache_mod.pool_cache_path("mine", _ctx()) + source.rename(source.parent / "other.json") + assert read_pool_cache("other", _ctx()) is None + + def test_a_mistyped_entry_is_a_miss_not_a_type_error(self) -> None: + # `dataclass` does not enforce types, so a well-shaped file with a stringly-typed + # capacity used to parse fine and only fail later, on `capacity - live`, as an + # uncaught TypeError in the middle of create(). + _save_cache("p", [_cached_host()]) + path = cache_mod.pool_cache_path("p", _ctx()) + data = json.loads(path.read_text()) + data["hosts"][0]["capacity"] = "50" + path.write_text(json.dumps(data)) + assert read_pool_cache("p", _ctx()) is None + + def test_implausible_values_are_a_miss(self) -> None: + for field, value in ( + ("capacity", True), # `bool` is an `int`, but not a count + ("live", -1), + ("nonce", "not-hex"), + ("nonce", NONCE[:8]), + ("job_id", "../../evil"), + ("owner", "user/../.."), + ("base_url", 1234), + ("updated_at", "yesterday"), + ): + _save_cache("p", [_cached_host()]) + path = cache_mod.pool_cache_path("p", _ctx()) + data = json.loads(path.read_text()) + data["hosts"][0][field] = value + path.write_text(json.dumps(data)) + assert read_pool_cache("p", _ctx()) is None, f"{field}={value!r} should be a miss" + + for field, value in (("sandboxes_per_host", 0), ("sandboxes_per_host", "4"), ("max_hosts", -2), ("image", 7)): + _save_cache("p", [_cached_host()]) + path = cache_mod.pool_cache_path("p", _ctx()) + data = json.loads(path.read_text()) + data[field] = value + path.write_text(json.dumps(data)) + assert read_pool_cache("p", _ctx()) is None, f"{field}={value!r} should be a miss" + + def test_a_mistyped_entry_does_not_break_a_connect(self, monkeypatch) -> None: + # The whole point: the failure mode is the documented cache miss (here, the cold path + # finding no host) rather than a TypeError surfacing from inside create(). + _save_cache("p", [_cached_host()]) + path = cache_mod.pool_cache_path("p", _ctx()) + data = json.loads(path.read_text()) + data["hosts"][0]["capacity"] = "50" + path.write_text(json.dumps(data)) + monkeypatch.setattr(sandbox_mod.HfApi, "list_jobs", _fake_list_jobs([])) + with pytest.raises(SandboxError, match="No running host found"): + SandboxPool.connect("p", token="hf_test") + + +class TestCachedHostAdmission: + """Where a cached host says it lives decides where the HF bearer and the host token go, + before any API call could contradict it. So the URL has to be one that only the job the + entry names could have: HTTPS, on that job's own proxy hostname.""" + + def test_a_loopback_url_is_refused(self) -> None: + assert sandbox_mod._cached_host_rejection(_cached_host("h1", url="http://127.0.0.1:1234")) is not None + + def test_plain_http_is_refused(self) -> None: + reason = sandbox_mod._cached_host_rejection( + _cached_host("h1", url=f"http://h1--{SANDBOX_SERVER_PORT}.hf.jobs") + ) + assert "https" in (reason or "") + + def test_another_jobs_url_is_refused(self) -> None: + entry = _cached_host("h1", url=f"https://h2--{SANDBOX_SERVER_PORT}.hf.jobs") + assert sandbox_mod._cached_host_rejection(entry) is not None + + def test_another_port_is_refused(self) -> None: + assert sandbox_mod._cached_host_rejection(_cached_host("h1", url="https://h1--8080.hf.jobs")) is not None + + def test_the_jobs_url_of_the_entrys_own_job_is_accepted(self) -> None: + assert sandbox_mod._cached_host_rejection(_cached_host("h1")) is None + # The domain is not hard-coded, so staging endpoints keep working. + entry = _cached_host("h1", url=f"https://h1--{SANDBOX_SERVER_PORT}.staging.hf.jobs/") + assert sandbox_mod._cached_host_rejection(entry) is None + + def test_a_cache_naming_a_local_server_makes_no_request_to_it(self, monkeypatch) -> None: + # The reproducer: a cache entry pointing at 127.0.0.1 used to receive the full bearer + # and sandbox token on the first create. The fake records every request it gets. + url, listener = _spawn_fake() + _save_cache("pool-local", [_cached_host("hostA", url=url)]) + monkeypatch.setattr(sandbox_mod.HfApi, "list_jobs", _fake_list_jobs([])) + + pool = SandboxPool.connect("pool-local", token="hf_test") # config from the cache, no HTTP + with pytest.raises(SandboxError, match="No running host found"): + pool.create() # the host is never seeded, so discovery finds nothing + assert listener.requests == [] + + def test_a_cross_namespace_connect_does_not_reach_the_other_namespaces_hosts(self, monkeypatch) -> None: + # `connect(pool, namespace="org-b")` used to be served org-a's cached hosts, because + # the cache was keyed by pool id alone and its namespace was read as configuration. + url, listener = _spawn_fake() + _save_cache("pool-x", [_cached_host("hostA", url=url)], context=_ctx("org-a")) + monkeypatch.setattr(sandbox_mod.HfApi, "list_jobs", _fake_list_jobs([])) + + with pytest.raises(SandboxError, match="No running host found"): + SandboxPool.connect("pool-x", namespace="org-b", token="hf_test") + assert listener.requests == [] + + def test_a_connect_without_a_namespace_does_not_inherit_the_cached_one(self, monkeypatch) -> None: + # It used to: `namespace=cache.namespace if namespace is None else namespace`. + url, listener = _spawn_fake() + _save_cache("pool-x", [_cached_host("hostA", url=url)], context=_ctx("org-a")) + monkeypatch.setattr(sandbox_mod.HfApi, "list_jobs", _fake_list_jobs([])) + + with pytest.raises(SandboxError, match="No running host found"): + SandboxPool.connect("pool-x", token="hf_test") + assert listener.requests == [] + + def test_an_inadmissible_url_is_pruned_from_the_cache(self, monkeypatch) -> None: + _save_cache("pool-local", [_cached_host("hostA", url="http://127.0.0.1:1234")]) + monkeypatch.setattr(sandbox_mod.HfApi, "list_jobs", _fake_list_jobs([])) + pool = SandboxPool.connect("pool-local", token="hf_test") + pool._seed_hosts_from_cache() + assert pool._hosts == [] + pool._save_cache() + cache = read_pool_cache("pool-local", _ctx()) + assert cache is not None and cache.hosts == [] + + +class TestCachedHostFreshness: + """A cached host is credited without a round-trip only while the entry is fresh -- written + by this same principal minutes ago, which is the `pool create` -> `create --pool` case the + cache exists for. Past that, it is a hint to be checked against the Jobs API.""" + + def _pool(self, name: str = "p1") -> SandboxPool: + # `_connect_mode` skips the constructor warm-up, so seeding runs in isolation. + return SandboxPool(image="python:3.12", flavor="cpu-basic", name=name, token="hf_test", _connect_mode=True) + + def test_a_fresh_entry_is_seeded_with_no_round_trip(self, monkeypatch) -> None: + _save_cache("p1", [_cached_host("hostA")]) + monkeypatch.setattr( + sandbox_mod.HfApi, "inspect_job", MagicMock(side_effect=AssertionError("must not inspect")) + ) + pool = self._pool() + pool._seed_hosts_from_cache() + assert [host.job_id for host in pool._hosts] == ["hostA"] + + def test_an_aged_entry_is_confirmed_against_the_jobs_api_first(self, monkeypatch) -> None: + _save_cache("p1", [_cached_host("hostA", age=HOST_TRUST_TTL + 60)]) + inspect = MagicMock(return_value=_pool_host_job("hostA", pool_name="p1")) + monkeypatch.setattr(sandbox_mod.HfApi, "inspect_job", inspect) + pool = self._pool() + pool._seed_hosts_from_cache() + assert [host.job_id for host in pool._hosts] == ["hostA"] + inspect.assert_called_once() + + def test_an_aged_entry_the_job_no_longer_backs_is_dropped(self, monkeypatch) -> None: + cases = { + "not running": lambda job: setattr(job.status, "stage", "CANCELED"), + "another pool": lambda job: job.labels.update({POOL_LABEL: "someone-elses-pool"}), + "another nonce": lambda job: job.labels.update({NONCE_LABEL: "ab" * 16}), + "another principal": lambda job: setattr(job.initiator, "id", "principal-attacker"), + "another url": lambda job: setattr(job.status, "expose_urls", ["https://elsewhere--49983.hf.jobs"]), + } + for label, mutate in cases.items(): + job = _pool_host_job("hostA", pool_name="p1") + mutate(job) + monkeypatch.setattr(sandbox_mod.HfApi, "inspect_job", MagicMock(return_value=job)) + _save_cache("p1", [_cached_host("hostA", age=HOST_TRUST_TTL + 60)]) + pool = self._pool() + pool._seed_hosts_from_cache() + assert pool._hosts == [], label + assert pool._dead_host_ids == {"hostA"}, label # and pruned on the next save + + def test_an_unreachable_jobs_api_preserves_the_entry(self, monkeypatch) -> None: + # A transient failure must not prune the entry: the host may well be fine. + _save_cache("p1", [_cached_host("hostA", age=HOST_TRUST_TTL + 60)]) + monkeypatch.setattr(sandbox_mod.HfApi, "inspect_job", MagicMock(side_effect=OSError("network is down"))) + pool = self._pool() + with pytest.raises(SandboxError, match="retry later"): + pool._seed_hosts_from_cache() + assert pool._hosts == [] + assert pool._dead_host_ids == set() + + def test_identity_failure_preserves_cache_and_can_be_retried(self, monkeypatch): + _save_cache("p1", [_cached_host("hostA", age=HOST_TRUST_TTL + 60)]) + monkeypatch.setattr( + sandbox_mod.HfApi, "whoami", MagicMock(side_effect=[OSError("offline"), {"id": PRINCIPAL_ID}]) + ) + monkeypatch.setattr( + sandbox_mod.HfApi, "inspect_job", MagicMock(return_value=_pool_host_job("hostA", pool_name="p1")) + ) + pool = self._pool() + with pytest.raises(SandboxError, match="current user"): + pool.create() + assert read_pool_cache("p1", pool._cache_context) is not None + assert pool._dead_host_ids == set() + pool._seed_hosts_from_cache() + assert pool.host_ids == ["hostA"] + + def test_missing_job_is_pruned(self, monkeypatch): + _save_cache("p1", [_cached_host("hostA", age=HOST_TRUST_TTL + 60)]) + response = httpx.Response(404, request=httpx.Request("GET", "https://huggingface.co/api/jobs/user/hostA")) + monkeypatch.setattr( + sandbox_mod.HfApi, + "inspect_job", + MagicMock(side_effect=httpx.HTTPStatusError("missing", request=response.request, response=response)), + ) + pool = self._pool() + pool._seed_hosts_from_cache() + assert pool._dead_host_ids == {"hostA"} + + +class TestTransportHardening: + """The client's own end of the protocol: no redirect-following on a credentialed client, + and no local file left mangled by a transfer.""" + + def test_a_redirect_is_not_followed(self, fake_server: str) -> None: + # Both credentials ride on every request, and httpx re-sends `Authorization` on a + # same-scheme redirect, so a 302 must be surfaced rather than chased. + elsewhere, listener = _spawn_fake() + _FakeServer.redirect_to = elsewhere + "/v1/sandboxes" + server = _make_server(fake_server) + response = server.request("GET", "/v1/redirect") + assert response.status_code == 302 + assert listener.requests == [] + + def test_download_replaces_a_symlink_instead_of_writing_through_it(self, fake_server: str, tmp_path) -> None: + sentinel = tmp_path / "precious" + sentinel.write_text("do not touch") + destination = tmp_path / "download.txt" + destination.symlink_to(sentinel) + + _make_sandbox(fake_server).files.download("/x", destination) + assert sentinel.read_text() == "do not touch" + assert not destination.is_symlink() + assert destination.read_bytes() == b"hello" + + def test_a_failed_download_leaves_nothing_behind(self, fake_server: str, tmp_path, monkeypatch) -> None: + sandbox = _make_sandbox(fake_server) + monkeypatch.setattr(sandbox, "_stream", MagicMock(side_effect=SandboxError("connection lost"))) + destination = tmp_path / "downloads" / "download.txt" + destination.parent.mkdir() + with pytest.raises(SandboxError): + sandbox.files.download("/x", destination) + assert list(destination.parent.iterdir()) == [] # no truncated file, no leftover temp file + + def test_upload_reads_the_file_it_measured(self, fake_server: str, tmp_path) -> None: + # One descriptor is opened, `fstat`ed and streamed, instead of stat-then-reopen. A + # symlinked source still works: `hf_hub_download` returns one. + source = tmp_path / "data.bin" + source.write_bytes(b"x" * 1024) + link = tmp_path / "link.bin" + link.symlink_to(source) + _make_sandbox(fake_server).files.upload(link, "/dest") + assert sum(len(body) for _, body in _FakeServer.writes) == 1024