Skip to content
Prev Previous commit
Next Next commit
align fetch_job_metrics with fetch_job_logs (follow param)
  • Loading branch information
hanouticelina committed Oct 5, 2026
commit 59210e2f44c9e7a0f2b164be5dde4cf637c0785c
13 changes: 2 additions & 11 deletions src/huggingface_hub/cli/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,6 @@ def _format_job_value(key: str, value: Any, from_script: Collection[str]) -> str


STATS_UPDATE_MIN_INTERVAL = 0.1 # we set a limit here since there is one update per second per job
STATS_SNAPSHOT_TIMEOUT = 10 # max seconds to wait for a first sample in snapshot mode

# Common job-related options
ImageArg = Annotated[
Expand Down Expand Up @@ -965,16 +964,8 @@ def jobs_stats(
def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None:
return next(iter(api.fetch_job_metrics(job_id=job_id, namespace=namespace)), None)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

# The metrics stream retries while the job is RUNNING, even if no sample ever comes, so we bound the wait.
with multiprocessing.pool.ThreadPool(len(job_ids)) as pool:
pending = [pool.apply_async(fetch_latest_metrics, (job_id,)) for job_id in job_ids]
deadline = time.monotonic() + STATS_SNAPSHOT_TIMEOUT
latest_metrics = []
for result in pending:
try:
latest_metrics.append(result.get(timeout=max(0.0, deadline - time.monotonic())))
except multiprocessing.TimeoutError:
latest_metrics.append(None)
latest_metrics = pool.map(fetch_latest_metrics, job_ids)
items = []
for job_id, metrics in zip(job_ids, latest_metrics):
if metrics is None:
Expand Down Expand Up @@ -1004,7 +995,7 @@ def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None:
kwargs_list = [
{
"job_id": job_id,
"metrics_stream": api.fetch_job_metrics(job_id=job_id, namespace=namespace),
"metrics_stream": api.fetch_job_metrics(job_id=job_id, namespace=namespace, follow=True),
"table_headers": table_headers,
}
for job_id in job_ids
Expand Down
21 changes: 18 additions & 3 deletions src/huggingface_hub/hf_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -8503,7 +8503,8 @@ def _stream_sse_events(
# Drop params after the first attempt: the start_event_idx dedup
# requires a stable replay prefix, which `tail` would break.
params = None
if on_iteration_end is not None and on_iteration_end():
# no-follow: a single attempt, never loop back
if not follow or (on_iteration_end is not None and on_iteration_end()):
break

def _fetch_space_logs_sse(
Expand Down Expand Up @@ -12162,10 +12163,11 @@ def fetch_job_metrics(
*,
job_id: str,
namespace: str | None = None,
follow: bool = False,
token: bool | str | None = None,
) -> Iterable[dict[str, Any]]:
"""
Fetch all the live metrics from a compute Job on Hugging Face infrastructure.
Fetch the live metrics from a compute Job on Hugging Face infrastructure.

Args:
job_id (`str`):
Expand All @@ -12174,6 +12176,10 @@ def fetch_job_metrics(
namespace (`str`, *optional*):
The namespace where the Job is running. Defaults to the current user's namespace.

follow (`bool`, *optional*):
If `True`, stream metrics in real-time until the job completes (blocking).
If `False` (default), fetch only the current metrics and return (non-blocking).

token (`bool` or `str`, *optional*):
A valid user access token. If not provided, the locally saved token will be used, which is the
recommended authentication method. Set to `False` to disable authentication.
Expand Down Expand Up @@ -12202,6 +12208,10 @@ def fetch_job_metrics(
},
"replica": "57vr7"
}

>>> # Stream metrics until the job completes
>>> for metrics in fetch_job_metrics(job_id=job.id, follow=True):
... print(metrics)
```
"""
# - there is one "metric" event every second, like this:
Expand All @@ -12213,15 +12223,20 @@ def fetch_job_metrics(
# - ChunkedEncodingError can happen in case of stopped logging in the middle of streaming
# - there is a ": keep-alive" every 30 seconds
seconds_between_events = 1
yield from self._fetch_running_job_sse(
metrics_stream = self._fetch_running_job_sse(
job_id=job_id,
route="metrics",
timeout=10 * seconds_between_events,
skip_previous_events_on_retry=False,
tolerated_status_codes=(500,),
namespace=namespace,
token=token,
follow=follow,
)
if follow:
yield from metrics_stream
else: # the stream has no history: its first sample is the current one
yield from itertools.islice(metrics_stream, 1)

def list_jobs(
self,
Expand Down
Loading