Skip to content
Next Next commit
[CLI] Snapshot by default in hf jobs stats, add -f/--follow
  • Loading branch information
hanouticelina committed Oct 5, 2026
commit f41b369f65f6356f10658a38abf8a9a8e80ec66d
2 changes: 2 additions & 0 deletions docs/source/en/guides/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -1892,6 +1892,8 @@ This command runs the job and shows the logs. You can pass `--detach` to run the
>>> hf jobs stats
# View resources usage stats and metrics of some jobs
>>> hf jobs stats [job_ids]...
# Follow live stats until the jobs complete
>>> hf jobs stats -f [job_ids]...

# Cancel a job
>>> hf jobs cancel <job_id>
Expand Down
10 changes: 8 additions & 2 deletions docs/source/en/package_reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -2314,7 +2314,7 @@ $ hf jobs [OPTIONS] COMMAND [ARGS]...
* `run`: Run a Job.
* `scheduled`: Create and manage scheduled Jobs on the Hub.
* `ssh`: SSH into a running Job.
* `stats`: Fetch the resource usage statistics and metrics of Jobs
* `stats`: Fetch the resource usage statistics and metrics of Jobs.
* `uv`: Run UV scripts (Python with inline dependencies) on HF infrastructure.
* `wait`: Wait for one or more Jobs to reach a terminal state.

Expand Down Expand Up @@ -3045,7 +3045,10 @@ Learn more

### `hf jobs stats`

Fetch the resource usage statistics and metrics of Jobs
Fetch the resource usage statistics and metrics of Jobs.

By default, prints a snapshot of the current stats and exits (non-blocking).
Use --follow/-f to display live stats until the Jobs complete.

**Usage**:

Expand All @@ -3059,12 +3062,15 @@ $ hf jobs stats [OPTIONS] [JOB_IDS]...

**Options**:

* `-f, --follow`: Follow stats output (live view until the Jobs complete). Without this flag, a single snapshot is printed.
* `--namespace TEXT`: The namespace where the job will be running. Defaults to the current user's namespace.
* `--token TEXT`: A User Access Token generated from https://huggingface.co/settings/tokens.
* `--help`: Show this message and exit.

Examples
$ hf jobs stats
$ hf jobs stats <job_id>
$ hf jobs stats -f <job_id>

Learn more
Use `hf <command> --help` for more information about a command.
Expand Down
96 changes: 72 additions & 24 deletions src/huggingface_hub/cli/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
typer_factory,
)
from ._framework import Argument, Option
from ._output import _dataclass_to_dict, out
from ._output import OutputFormat, _dataclass_to_dict, out
from ._uv_script_header import TABLE_NAME, UvScriptHeader, load_uv_script


Expand Down Expand Up @@ -868,41 +868,68 @@ def _clear_line(n: int) -> None:
print(LINE_UP, end=LINE_CLEAR)


def _format_job_stats_rows(job_id: str, metrics: dict[str, Any], table_headers: list[str]) -> list[list[str | int]]:
row: list[str | int] = [
job_id,
f"{metrics['cpu_usage_pct']}%",
round(metrics["cpu_millicores"] / 1000.0, 1),
f"{round(100 * metrics['memory_used_bytes'] / metrics['memory_total_bytes'], 2)}%",
f"{_format_size(metrics['memory_used_bytes'])}B / {_format_size(metrics['memory_total_bytes'])}B",
f"{_format_size(metrics['rx_bps'])}bps / {_format_size(metrics['tx_bps'])}bps",
]
if metrics["gpus"] and isinstance(metrics["gpus"], dict):
rows: list[list[str | int]] = [row] + [[""] * len(row) for _ in range(len(metrics["gpus"]) - 1)]
for row, gpu_id in zip(rows, sorted(metrics["gpus"])):
gpu = metrics["gpus"][gpu_id]
row += [
f"{gpu['utilization']}%",
f"{round(100 * gpu['memory_used_bytes'] / gpu['memory_total_bytes'], 2)}%",
f"{_format_size(gpu['memory_used_bytes'])}B / {_format_size(gpu['memory_total_bytes'])}B",
]
return rows
row += ["N/A"] * (len(table_headers) - len(row))
return [row]


def _get_jobs_stats_rows(
job_id: str, metrics_stream: Iterable[dict[str, Any]], table_headers: list[str]
) -> Iterable[tuple[bool, str, list[list[str | int]]]]:
for metrics in metrics_stream:
row = [
job_id,
f"{metrics['cpu_usage_pct']}%",
round(metrics["cpu_millicores"] / 1000.0, 1),
f"{round(100 * metrics['memory_used_bytes'] / metrics['memory_total_bytes'], 2)}%",
f"{_format_size(metrics['memory_used_bytes'])}B / {_format_size(metrics['memory_total_bytes'])}B",
f"{_format_size(metrics['rx_bps'])}bps / {_format_size(metrics['tx_bps'])}bps",
]
if metrics["gpus"] and isinstance(metrics["gpus"], dict):
rows = [row] + [[""] * len(row) for _ in range(len(metrics["gpus"]) - 1)]
for row, gpu_id in zip(rows, sorted(metrics["gpus"])):
gpu = metrics["gpus"][gpu_id]
row += [
f"{gpu['utilization']}%",
f"{round(100 * gpu['memory_used_bytes'] / gpu['memory_total_bytes'], 2)}%",
f"{_format_size(gpu['memory_used_bytes'])}B / {_format_size(gpu['memory_total_bytes'])}B",
]
else:
row += ["N/A"] * (len(table_headers) - len(row))
rows = [row]
yield False, job_id, rows
yield False, job_id, _format_job_stats_rows(job_id, metrics, table_headers)
yield True, job_id, []


@jobs_cli.command("stats", examples=["hf jobs stats <job_id>"])
@jobs_cli.command(
"stats",
examples=[
"hf jobs stats",
"hf jobs stats <job_id>",
"hf jobs stats -f <job_id>",
],
)
def jobs_stats(
job_ids: JobIdsArg = None,
follow: Annotated[
bool,
Option(
"-f",
"--follow",
help="Follow stats output (live view until the Jobs complete). Without this flag, a single snapshot is printed.",
),
] = False,
namespace: NamespaceOpt = None,
token: TokenOpt = None,
) -> None:
"""Fetch the resource usage statistics and metrics of Jobs"""
"""Fetch the resource usage statistics and metrics of Jobs.

By default, prints a snapshot of the current stats and exits (non-blocking).
Use --follow/-f to display live stats until the Jobs complete.
"""
if follow and out.mode != OutputFormat.human:
raise CLIError(
f"`--follow` displays a live terminal view and is not supported with '{out.mode.value}' output."
" Run without `--follow` to get a snapshot."
)
if job_ids is not None:
parsed_ids = []
for job_id in job_ids:
Expand Down Expand Up @@ -932,6 +959,27 @@ def jobs_stats(
"GPU MEM %",
"GPU MEM USAGE",
]
if not follow:

def fetch_latest_metrics(job_id: str) -> dict[str, Any] | None:
return next(iter(api.fetch_job_metrics(job_id=job_id, namespace=namespace)), None)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

with multiprocessing.pool.ThreadPool(len(job_ids)) as pool:
latest_metrics = pool.map(fetch_latest_metrics, job_ids)
items = []
for job_id, metrics in zip(job_ids, latest_metrics):
if metrics is None:
out.warning(f"No stats available for Job {job_id}. Is it running?")
else:
items.append({"job_id": job_id, **metrics})
if out.mode == OutputFormat.human and items:
rows = [row for item in items for row in _format_job_stats_rows(item["job_id"], item, table_headers)]
out.text(_tabulate(rows, headers=table_headers))
job_refs = " ".join(f"{namespace}/{item['job_id']}" for item in items)
out.hint(f"Use `hf jobs stats -f {job_refs}` to follow live stats.")
else:
out.table(items, id_key="job_id")
return
with multiprocessing.pool.ThreadPool(len(job_ids)) as pool:
rows_per_job_id: dict[str, list[list[str | int]]] = {}
for job_id in job_ids:
Expand Down
Loading