From f41b369f65f6356f10658a38abf8a9a8e80ec66d Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Mon, 5 Oct 2026 17:14:48 +0200 Subject: [PATCH 1/5] [CLI] Snapshot by default in hf jobs stats, add -f/--follow --- docs/source/en/guides/cli.md | 2 + docs/source/en/package_reference/cli.md | 10 ++- src/huggingface_hub/cli/jobs.py | 96 ++++++++++++++++++------- 3 files changed, 82 insertions(+), 26 deletions(-) diff --git a/docs/source/en/guides/cli.md b/docs/source/en/guides/cli.md index d7d173528f..26cfb63f0b 100644 --- a/docs/source/en/guides/cli.md +++ b/docs/source/en/guides/cli.md @@ -1892,6 +1892,8 @@ This command runs the job and shows the logs. You can pass `--detach` to run the >>> hf jobs stats # View resources usage stats and metrics of some jobs >>> hf jobs stats [job_ids]... +# Follow live stats until the jobs complete +>>> hf jobs stats -f [job_ids]... # Cancel a job >>> hf jobs cancel diff --git a/docs/source/en/package_reference/cli.md b/docs/source/en/package_reference/cli.md index 917ef0a436..920464c713 100644 --- a/docs/source/en/package_reference/cli.md +++ b/docs/source/en/package_reference/cli.md @@ -2314,7 +2314,7 @@ $ hf jobs [OPTIONS] COMMAND [ARGS]... * `run`: Run a Job. * `scheduled`: Create and manage scheduled Jobs on the Hub. * `ssh`: SSH into a running Job. -* `stats`: Fetch the resource usage statistics and metrics of Jobs +* `stats`: Fetch the resource usage statistics and metrics of Jobs. * `uv`: Run UV scripts (Python with inline dependencies) on HF infrastructure. * `wait`: Wait for one or more Jobs to reach a terminal state. @@ -3045,7 +3045,10 @@ Learn more ### `hf jobs stats` -Fetch the resource usage statistics and metrics of Jobs +Fetch the resource usage statistics and metrics of Jobs. + +By default, prints a snapshot of the current stats and exits (non-blocking). +Use --follow/-f to display live stats until the Jobs complete. **Usage**: @@ -3059,12 +3062,15 @@ $ hf jobs stats [OPTIONS] [JOB_IDS]... **Options**: +* `-f, --follow`: Follow stats output (live view until the Jobs complete). Without this flag, a single snapshot is printed. * `--namespace TEXT`: The namespace where the job will be running. Defaults to the current user's namespace. * `--token TEXT`: A User Access Token generated from https://huggingface.co/settings/tokens. * `--help`: Show this message and exit. Examples + $ hf jobs stats $ hf jobs stats + $ hf jobs stats -f Learn more Use `hf --help` for more information about a command. diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 331f159f5d..3bfd4101ee 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -62,7 +62,7 @@ typer_factory, ) from ._framework import Argument, Option -from ._output import _dataclass_to_dict, out +from ._output import OutputFormat, _dataclass_to_dict, out from ._uv_script_header import TABLE_NAME, UvScriptHeader, load_uv_script @@ -868,41 +868,68 @@ def _clear_line(n: int) -> None: print(LINE_UP, end=LINE_CLEAR) +def _format_job_stats_rows(job_id: str, metrics: dict[str, Any], table_headers: list[str]) -> list[list[str | int]]: + row: list[str | int] = [ + job_id, + f"{metrics['cpu_usage_pct']}%", + round(metrics["cpu_millicores"] / 1000.0, 1), + f"{round(100 * metrics['memory_used_bytes'] / metrics['memory_total_bytes'], 2)}%", + f"{_format_size(metrics['memory_used_bytes'])}B / {_format_size(metrics['memory_total_bytes'])}B", + f"{_format_size(metrics['rx_bps'])}bps / {_format_size(metrics['tx_bps'])}bps", + ] + if metrics["gpus"] and isinstance(metrics["gpus"], dict): + rows: list[list[str | int]] = [row] + [[""] * len(row) for _ in range(len(metrics["gpus"]) - 1)] + for row, gpu_id in zip(rows, sorted(metrics["gpus"])): + gpu = metrics["gpus"][gpu_id] + row += [ + f"{gpu['utilization']}%", + f"{round(100 * gpu['memory_used_bytes'] / gpu['memory_total_bytes'], 2)}%", + f"{_format_size(gpu['memory_used_bytes'])}B / {_format_size(gpu['memory_total_bytes'])}B", + ] + return rows + row += ["N/A"] * (len(table_headers) - len(row)) + return [row] + + def _get_jobs_stats_rows( job_id: str, metrics_stream: Iterable[dict[str, Any]], table_headers: list[str] ) -> Iterable[tuple[bool, str, list[list[str | int]]]]: for metrics in metrics_stream: - row = [ - job_id, - f"{metrics['cpu_usage_pct']}%", - round(metrics["cpu_millicores"] / 1000.0, 1), - f"{round(100 * metrics['memory_used_bytes'] / metrics['memory_total_bytes'], 2)}%", - f"{_format_size(metrics['memory_used_bytes'])}B / {_format_size(metrics['memory_total_bytes'])}B", - f"{_format_size(metrics['rx_bps'])}bps / {_format_size(metrics['tx_bps'])}bps", - ] - if metrics["gpus"] and isinstance(metrics["gpus"], dict): - rows = [row] + [[""] * len(row) for _ in range(len(metrics["gpus"]) - 1)] - for row, gpu_id in zip(rows, sorted(metrics["gpus"])): - gpu = metrics["gpus"][gpu_id] - row += [ - f"{gpu['utilization']}%", - f"{round(100 * gpu['memory_used_bytes'] / gpu['memory_total_bytes'], 2)}%", - f"{_format_size(gpu['memory_used_bytes'])}B / {_format_size(gpu['memory_total_bytes'])}B", - ] - else: - row += ["N/A"] * (len(table_headers) - len(row)) - rows = [row] - yield False, job_id, rows + yield False, job_id, _format_job_stats_rows(job_id, metrics, table_headers) yield True, job_id, [] -@jobs_cli.command("stats", examples=["hf jobs stats "]) +@jobs_cli.command( + "stats", + examples=[ + "hf jobs stats", + "hf jobs stats ", + "hf jobs stats -f ", + ], +) def jobs_stats( job_ids: JobIdsArg = None, + follow: Annotated[ + bool, + Option( + "-f", + "--follow", + help="Follow stats output (live view until the Jobs complete). Without this flag, a single snapshot is printed.", + ), + ] = False, namespace: NamespaceOpt = None, token: TokenOpt = None, ) -> None: - """Fetch the resource usage statistics and metrics of Jobs""" + """Fetch the resource usage statistics and metrics of Jobs. + + By default, prints a snapshot of the current stats and exits (non-blocking). + Use --follow/-f to display live stats until the Jobs complete. + """ + if follow and out.mode != OutputFormat.human: + raise CLIError( + f"`--follow` displays a live terminal view and is not supported with '{out.mode.value}' output." + " Run without `--follow` to get a snapshot." + ) if job_ids is not None: parsed_ids = [] for job_id in job_ids: @@ -932,6 +959,27 @@ def jobs_stats( "GPU MEM %", "GPU MEM USAGE", ] + if not follow: + + def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None: + return next(iter(api.fetch_job_metrics(job_id=job_id, namespace=namespace)), None) + + with multiprocessing.pool.ThreadPool(len(job_ids)) as pool: + latest_metrics = pool.map(fetch_latest_metrics, job_ids) + items = [] + for job_id, metrics in zip(job_ids, latest_metrics): + if metrics is None: + out.warning(f"No stats available for Job {job_id}. Is it running?") + else: + items.append({"job_id": job_id, **metrics}) + if out.mode == OutputFormat.human and items: + rows = [row for item in items for row in _format_job_stats_rows(item["job_id"], item, table_headers)] + out.text(_tabulate(rows, headers=table_headers)) + job_refs = " ".join(f"{namespace}/{item['job_id']}" for item in items) + out.hint(f"Use `hf jobs stats -f {job_refs}` to follow live stats.") + else: + out.table(items, id_key="job_id") + return with multiprocessing.pool.ThreadPool(len(job_ids)) as pool: rows_per_job_id: dict[str, list[list[str | int]]] = {} for job_id in job_ids: From 13cf37c2015dc095e89ed3a84da04d1d247a0245 Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Mon, 5 Oct 2026 17:43:06 +0200 Subject: [PATCH 2/5] fix snapshot hanging on jobs that send no metrics --- src/huggingface_hub/cli/jobs.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 3bfd4101ee..641deee433 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -437,6 +437,7 @@ def _format_job_value(key: str, value: Any, from_script: Collection[str]) -> str STATS_UPDATE_MIN_INTERVAL = 0.1 # we set a limit here since there is one update per second per job +STATS_SNAPSHOT_TIMEOUT = 10 # max seconds to wait for a first sample in snapshot mode # Common job-related options ImageArg = Annotated[ @@ -964,8 +965,16 @@ def jobs_stats( def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None: return next(iter(api.fetch_job_metrics(job_id=job_id, namespace=namespace)), None) + # The metrics stream retries while the job is RUNNING, even if no sample ever comes, so we bound the wait. with multiprocessing.pool.ThreadPool(len(job_ids)) as pool: - latest_metrics = pool.map(fetch_latest_metrics, job_ids) + pending = [pool.apply_async(fetch_latest_metrics, (job_id,)) for job_id in job_ids] + deadline = time.monotonic() + STATS_SNAPSHOT_TIMEOUT + latest_metrics = [] + for result in pending: + try: + latest_metrics.append(result.get(timeout=max(0.0, deadline - time.monotonic()))) + except multiprocessing.TimeoutError: + latest_metrics.append(None) items = [] for job_id, metrics in zip(job_ids, latest_metrics): if metrics is None: From 59210e2f44c9e7a0f2b164be5dde4cf637c0785c Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Mon, 5 Oct 2026 18:13:44 +0200 Subject: [PATCH 3/5] align fetch_job_metrics with fetch_job_logs (follow param) --- src/huggingface_hub/cli/jobs.py | 13 ++----------- src/huggingface_hub/hf_api.py | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 641deee433..0f0f6e54b4 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -437,7 +437,6 @@ def _format_job_value(key: str, value: Any, from_script: Collection[str]) -> str STATS_UPDATE_MIN_INTERVAL = 0.1 # we set a limit here since there is one update per second per job -STATS_SNAPSHOT_TIMEOUT = 10 # max seconds to wait for a first sample in snapshot mode # Common job-related options ImageArg = Annotated[ @@ -965,16 +964,8 @@ def jobs_stats( def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None: return next(iter(api.fetch_job_metrics(job_id=job_id, namespace=namespace)), None) - # The metrics stream retries while the job is RUNNING, even if no sample ever comes, so we bound the wait. with multiprocessing.pool.ThreadPool(len(job_ids)) as pool: - pending = [pool.apply_async(fetch_latest_metrics, (job_id,)) for job_id in job_ids] - deadline = time.monotonic() + STATS_SNAPSHOT_TIMEOUT - latest_metrics = [] - for result in pending: - try: - latest_metrics.append(result.get(timeout=max(0.0, deadline - time.monotonic()))) - except multiprocessing.TimeoutError: - latest_metrics.append(None) + latest_metrics = pool.map(fetch_latest_metrics, job_ids) items = [] for job_id, metrics in zip(job_ids, latest_metrics): if metrics is None: @@ -1004,7 +995,7 @@ def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None: kwargs_list = [ { "job_id": job_id, - "metrics_stream": api.fetch_job_metrics(job_id=job_id, namespace=namespace), + "metrics_stream": api.fetch_job_metrics(job_id=job_id, namespace=namespace, follow=True), "table_headers": table_headers, } for job_id in job_ids diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 20f5d243d4..3eed7c9bec 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -8503,7 +8503,8 @@ def _stream_sse_events( # Drop params after the first attempt: the start_event_idx dedup # requires a stable replay prefix, which `tail` would break. params = None - if on_iteration_end is not None and on_iteration_end(): + # no-follow: a single attempt, never loop back + if not follow or (on_iteration_end is not None and on_iteration_end()): break def _fetch_space_logs_sse( @@ -12162,10 +12163,11 @@ def fetch_job_metrics( *, job_id: str, namespace: str | None = None, + follow: bool = False, token: bool | str | None = None, ) -> Iterable[dict[str, Any]]: """ - Fetch all the live metrics from a compute Job on Hugging Face infrastructure. + Fetch the live metrics from a compute Job on Hugging Face infrastructure. Args: job_id (`str`): @@ -12174,6 +12176,10 @@ def fetch_job_metrics( namespace (`str`, *optional*): The namespace where the Job is running. Defaults to the current user's namespace. + follow (`bool`, *optional*): + If `True`, stream metrics in real-time until the job completes (blocking). + If `False` (default), fetch only the current metrics and return (non-blocking). + token (`bool` or `str`, *optional*): A valid user access token. If not provided, the locally saved token will be used, which is the recommended authentication method. Set to `False` to disable authentication. @@ -12202,6 +12208,10 @@ def fetch_job_metrics( }, "replica": "57vr7" } + + >>> # Stream metrics until the job completes + >>> for metrics in fetch_job_metrics(job_id=job.id, follow=True): + ... print(metrics) ``` """ # - there is one "metric" event every second, like this: @@ -12213,7 +12223,7 @@ def fetch_job_metrics( # - ChunkedEncodingError can happen in case of stopped logging in the middle of streaming # - there is a ": keep-alive" every 30 seconds seconds_between_events = 1 - yield from self._fetch_running_job_sse( + metrics_stream = self._fetch_running_job_sse( job_id=job_id, route="metrics", timeout=10 * seconds_between_events, @@ -12221,7 +12231,12 @@ def fetch_job_metrics( tolerated_status_codes=(500,), namespace=namespace, token=token, + follow=follow, ) + if follow: + yield from metrics_stream + else: # the stream has no history: its first sample is the current one + yield from itertools.islice(metrics_stream, 1) def list_jobs( self, From 57dad5f7d9ed2a95cf0d9827b38ce3ef8ff55500 Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Mon, 5 Oct 2026 18:31:21 +0200 Subject: [PATCH 4/5] add a hint when hf jobs stats -f ends --- src/huggingface_hub/cli/jobs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 0f0f6e54b4..4768d47e8a 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -1011,6 +1011,8 @@ def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None: total_rows = [row for job_id in rows_per_job_id for row in rows_per_job_id[job_id]] print(_tabulate(total_rows, headers=table_headers)) last_update_time = now + job_refs = " ".join(f"{namespace}/{job_id}" for job_id in job_ids) + out.hint(f"All Jobs stopped. Run `hf jobs inspect {job_refs}` to check their final status.") @jobs_cli.command( From cfab3e32d234505fc230b7058017cdf91b00260a Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Mon, 5 Oct 2026 18:38:30 +0200 Subject: [PATCH 5/5] fix stats -f end hint claiming jobs stopped --- src/huggingface_hub/cli/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 4768d47e8a..22c9ab6b76 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -1012,7 +1012,7 @@ def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None: print(_tabulate(total_rows, headers=table_headers)) last_update_time = now job_refs = " ".join(f"{namespace}/{job_id}" for job_id in job_ids) - out.hint(f"All Jobs stopped. Run `hf jobs inspect {job_refs}` to check their final status.") + out.hint(f"Stream ended. Run `hf jobs inspect {job_refs}` to check the final status (e.g. COMPLETED or ERROR).") @jobs_cli.command(