Skip to content
Prev Previous commit
Next Next commit
fix snapshot hanging on jobs that send no metrics
  • Loading branch information
hanouticelina committed Oct 5, 2026
commit 13cf37c2015dc095e89ed3a84da04d1d247a0245
11 changes: 10 additions & 1 deletion src/huggingface_hub/cli/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,7 @@ def _format_job_value(key: str, value: Any, from_script: Collection[str]) -> str


STATS_UPDATE_MIN_INTERVAL = 0.1 # we set a limit here since there is one update per second per job
STATS_SNAPSHOT_TIMEOUT = 10 # max seconds to wait for a first sample in snapshot mode

# Common job-related options
ImageArg = Annotated[
Expand Down Expand Up @@ -964,8 +965,16 @@ def jobs_stats(
def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None:
return next(iter(api.fetch_job_metrics(job_id=job_id, namespace=namespace)), None)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

# The metrics stream retries while the job is RUNNING, even if no sample ever comes, so we bound the wait.
with multiprocessing.pool.ThreadPool(len(job_ids)) as pool:
latest_metrics = pool.map(fetch_latest_metrics, job_ids)
pending = [pool.apply_async(fetch_latest_metrics, (job_id,)) for job_id in job_ids]
deadline = time.monotonic() + STATS_SNAPSHOT_TIMEOUT
latest_metrics = []
for result in pending:
try:
latest_metrics.append(result.get(timeout=max(0.0, deadline - time.monotonic())))
except multiprocessing.TimeoutError:
latest_metrics.append(None)
Comment thread
hanouticelina marked this conversation as resolved.
Resolved
items = []
for job_id, metrics in zip(job_ids, latest_metrics):
if metrics is None:
Expand Down
Loading