From 0707ad9375b410c92c0d31c09c6691845a7f2973 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Fri, 6 Feb 2026 13:11:35 +0000 Subject: [PATCH 1/5] Add `--follow/-f` and `--tail/-n` flags to `hf jobs logs` `hf jobs logs` currently streams logs indefinitely via SSE until a job completes, blocking the caller. This makes it harder to use for agents, CI scripts, and any programmatic consumer that needs to check logs without hanging. This follows Docker's convention: `docker logs` prints existing logs and exits by default, while `docker logs -f` opts into streaming. Similarly: - `hf jobs logs ` now prints available logs and exits (non-blocking) - `hf jobs logs -f ` streams until the job completes (blocking) - `hf jobs logs --tail N ` shows only the last N lines The Python API adds a `follow` parameter to `fetch_job_logs()` (default `True` for backward compatibility). The CLI defaults to `follow=False`. Co-Authored-By: Claude Opus 4.6 --- docs/source/en/package_reference/cli.md | 11 ++++- src/huggingface_hub/cli/jobs.py | 42 +++++++++++++++--- src/huggingface_hub/hf_api.py | 57 +++++++++++++++++++++++++ tests/test_cli.py | 55 +++++++++++++++++++++++- 4 files changed, 156 insertions(+), 9 deletions(-) diff --git a/docs/source/en/package_reference/cli.md b/docs/source/en/package_reference/cli.md index c7c44b10f7f..7f9cb29d474 100644 --- a/docs/source/en/package_reference/cli.md +++ b/docs/source/en/package_reference/cli.md @@ -1116,7 +1116,7 @@ $ hf jobs [OPTIONS] COMMAND [ARGS]... * `cancel`: Cancel a Job * `hardware`: List available hardware options for Jobs * `inspect`: Display detailed information on one or... -* `logs`: Fetch the logs of a Job +* `logs`: Fetch the logs of a Job. * `ps`: List Jobs. * `run`: Run a Job. * `scheduled`: Create and manage scheduled Jobs on the Hub. @@ -1203,7 +1203,10 @@ Learn more ### `hf jobs logs` -Fetch the logs of a Job +Fetch the logs of a Job. + +By default, prints currently available logs and exits (non-blocking). +Use --follow/-f to stream logs in real-time until the job completes. **Usage**: @@ -1217,12 +1220,16 @@ $ hf jobs logs [OPTIONS] JOB_ID **Options**: +* `-f, --follow`: Follow log output (stream until the job completes). Without this flag, only currently available logs are printed. +* `-n, --tail INTEGER`: Number of lines to show from the end of the logs. * `--namespace TEXT`: The namespace where the job will be running. Defaults to the current user's namespace. * `--token TEXT`: A User Access Token generated from https://huggingface.co/settings/tokens. * `--help`: Show this message and exit. Examples $ hf jobs logs + $ hf jobs logs -f + $ hf jobs logs --tail 20 Learn more Use `hf --help` for more information about a command. diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 7d3624e3f9d..7f8e20e4db6 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -20,9 +20,12 @@ # List running or completed jobs hf jobs ps [-a] [-f key=value] [--format TEMPLATE] - # Stream logs from a job + # Print logs from a job (non-blocking) hf jobs logs + # Stream logs from a job (blocking, like `docker logs -f`) + hf jobs logs -f + # Stream resources usage stats and metrics from a job hf jobs stats @@ -312,20 +315,47 @@ def jobs_run( if detach: return # Now let's stream the logs - for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name): + for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): print(log) -@jobs_cli.command("logs", examples=["hf jobs logs "]) +@jobs_cli.command( + "logs", examples=["hf jobs logs ", "hf jobs logs -f ", "hf jobs logs --tail 20 "] +) def jobs_logs( job_id: JobIdArg, + follow: Annotated[ + bool, + typer.Option( + "-f", + "--follow", + help="Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.", + ), + ] = False, + tail: Annotated[ + Optional[int], + typer.Option( + "-n", + "--tail", + help="Number of lines to show from the end of the logs.", + ), + ] = None, namespace: NamespaceOpt = None, token: TokenOpt = None, ) -> None: - """Fetch the logs of a Job""" + """Fetch the logs of a Job. + + By default, prints currently available logs and exits (non-blocking). + Use --follow/-f to stream logs in real-time until the job completes. + """ api = get_hf_api(token=token) try: - for log in api.fetch_job_logs(job_id=job_id, namespace=namespace): + logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow) + if tail is not None: + from collections import deque + + logs = deque(logs, maxlen=tail) + for log in logs: print(log) except HfHubHTTPError as e: status = e.response.status_code if e.response is not None else None @@ -729,7 +759,7 @@ def jobs_uv_run( if detach: return # Now let's stream the logs - for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name): + for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): print(log) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 292114b6db1..a4a84a0b713 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -10335,6 +10335,7 @@ def fetch_job_logs( *, job_id: str, namespace: Optional[str] = None, + follow: bool = True, token: Union[bool, str, None] = None, ) -> Iterable[str]: """ @@ -10347,6 +10348,11 @@ def fetch_job_logs( namespace (`str`, *optional*): The namespace where the Job is running. Defaults to the current user's namespace. + follow (`bool`, *optional*): + If `True` (default), stream logs in real-time until the job completes (blocking). + If `False`, fetch only the currently available logs and return immediately (non-blocking). + Use `follow=False` when calling from scripts, agents, or CI pipelines that should not block. + token `(Union[bool, str, None]`, *optional*): A valid user access token. If not provided, the locally saved token will be used, which is the recommended authentication method. Set to `False` to disable authentication. @@ -10360,8 +10366,16 @@ def fetch_job_logs( >>> for log in fetch_job_logs(job_id=job.id): ... print(log) Hello from HF compute! + + >>> # Non-blocking: fetch only currently available logs + >>> for log in fetch_job_logs(job_id=job.id, follow=False): + ... print(log) ``` """ + if not follow: + yield from self._fetch_job_logs_no_follow(job_id=job_id, namespace=namespace, token=token) + return + # - We need to retry because sometimes the /logs doesn't return logs when the job just started. # (for example it can return only two lines: one for "Job started" and one empty line) # - Timeouts can happen in case of build errors @@ -10385,6 +10399,49 @@ def fetch_job_logs( log = event["data"] yield log + def _fetch_job_logs_no_follow( + self, + *, + job_id: str, + namespace: Optional[str] = None, + token: Union[bool, str, None] = None, + ) -> Iterable[str]: + """Fetch only currently available logs and return immediately (non-blocking). + + Connects to the SSE log stream with a short read timeout. The server replays + all historical log lines quickly, then pauses waiting for new events (with + keep-alive every ~30s). When the short timeout fires without new data, we + treat that as "all buffered logs received" and return gracefully. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + + # Short timeout: the server replays historical logs quickly, then pauses. + # 5 seconds is enough to receive all buffered logs. + read_timeout = 5 + try: + with get_session().stream( + "GET", + f"{self.endpoint}/api/jobs/{namespace}/{job_id}/logs", + headers=self._build_hf_headers(token=token), + timeout=read_timeout, + ) as response: + hf_raise_for_status(response) + for line in response.iter_lines(): + if line and line.startswith("data: {"): + event = json.loads(line[len("data: ") :]) + if not event["data"].startswith("===== Job started"): + yield event["data"] + except (httpx.ReadTimeout, httpcore.ReadTimeout): + # ReadTimeout means we received all buffered logs and the server is waiting + # for new events - this is the expected exit path for no-follow mode. + pass + except httpx.NetworkError as err: + # NetworkError can wrap a TimeoutError from the underlying socket. + is_timeout = err.__context__ and isinstance(getattr(err.__context__, "__cause__", None), TimeoutError) + if not is_timeout: + raise + def fetch_job_metrics( self, *, diff --git a/tests/test_cli.py b/tests/test_cli.py index ef3216572e9..7792ae08587 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1966,4 +1966,57 @@ def test_run_fetches_logs_with_correct_namespace(self, runner: CliRunner) -> Non api.fetch_job_logs.return_value = iter(["log line 1"]) result = runner.invoke(app, ["jobs", "run", "ubuntu", "echo", "hello"]) assert result.exit_code == 0 - api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace="my-username") + api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace="my-username", follow=True) + + def test_logs_default_no_follow(self, runner: CliRunner) -> None: + """Test that `hf jobs logs ` defaults to follow=False (non-blocking, like `docker logs`).""" + with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls: + api = api_cls.return_value + api.fetch_job_logs.return_value = iter(["line 1", "line 2"]) + result = runner.invoke(app, ["jobs", "logs", "my-job-id"]) + assert result.exit_code == 0 + api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=False) + assert "line 1" in result.output + assert "line 2" in result.output + + def test_logs_follow_flag(self, runner: CliRunner) -> None: + """Test that `hf jobs logs -f ` passes follow=True.""" + with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls: + api = api_cls.return_value + api.fetch_job_logs.return_value = iter(["streaming line"]) + result = runner.invoke(app, ["jobs", "logs", "-f", "my-job-id"]) + assert result.exit_code == 0 + api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=True) + assert "streaming line" in result.output + + def test_logs_follow_long_flag(self, runner: CliRunner) -> None: + """Test that `hf jobs logs --follow ` passes follow=True.""" + with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls: + api = api_cls.return_value + api.fetch_job_logs.return_value = iter(["streaming line"]) + result = runner.invoke(app, ["jobs", "logs", "--follow", "my-job-id"]) + assert result.exit_code == 0 + api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=True) + + def test_logs_tail(self, runner: CliRunner) -> None: + """Test that `hf jobs logs --tail 2 ` shows only the last 2 lines.""" + with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls: + api = api_cls.return_value + api.fetch_job_logs.return_value = iter(["line 1", "line 2", "line 3", "line 4"]) + result = runner.invoke(app, ["jobs", "logs", "--tail", "2", "my-job-id"]) + assert result.exit_code == 0 + assert "line 1" not in result.output + assert "line 2" not in result.output + assert "line 3" in result.output + assert "line 4" in result.output + + def test_logs_tail_short_flag(self, runner: CliRunner) -> None: + """Test that `hf jobs logs -n 1 ` shows only the last line.""" + with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls: + api = api_cls.return_value + api.fetch_job_logs.return_value = iter(["line 1", "line 2", "line 3"]) + result = runner.invoke(app, ["jobs", "logs", "-n", "1", "my-job-id"]) + assert result.exit_code == 0 + assert "line 1" not in result.output + assert "line 2" not in result.output + assert "line 3" in result.output From 97f49fa4128abacdd57fcb9115f17662de074995 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Fri, 6 Feb 2026 14:05:29 +0000 Subject: [PATCH 2/5] Make `fetch_job_logs()` default to `follow=False` for consistency Per maintainer feedback, align the Python API default with the CLI: both now default to non-blocking (follow=False). The two internal call sites that need streaming (jobs_run, jobs_uv_run without --detach) already pass follow=True explicitly. Co-Authored-By: Claude Opus 4.6 --- src/huggingface_hub/hf_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index a4a84a0b713..a71f7b697c8 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -10335,7 +10335,7 @@ def fetch_job_logs( *, job_id: str, namespace: Optional[str] = None, - follow: bool = True, + follow: bool = False, token: Union[bool, str, None] = None, ) -> Iterable[str]: """ From 3af4150c89c5477cce804495dcdd4260dd39542a Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Mon, 9 Feb 2026 10:31:52 +0000 Subject: [PATCH 3/5] Move `deque` import to module level Co-Authored-By: Claude Opus 4.6 --- src/huggingface_hub/cli/jobs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 7f8e20e4db6..1cfd9b987c3 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -67,6 +67,7 @@ import os import shutil import time +from collections import deque from dataclasses import asdict from fnmatch import fnmatch from pathlib import Path @@ -352,8 +353,6 @@ def jobs_logs( try: logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow) if tail is not None: - from collections import deque - logs = deque(logs, maxlen=tail) for log in logs: print(log) From 80c38c7c1763ad171c932138c1ffb2637b8093ff Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Mon, 9 Feb 2026 10:33:21 +0000 Subject: [PATCH 4/5] Raise error when --follow and --tail are used together Co-Authored-By: Claude Opus 4.6 --- src/huggingface_hub/cli/jobs.py | 5 +++++ tests/test_cli.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index 1cfd9b987c3..9c434bdc8fd 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -349,6 +349,11 @@ def jobs_logs( By default, prints currently available logs and exits (non-blocking). Use --follow/-f to stream logs in real-time until the job completes. """ + if follow and tail is not None: + raise CLIError( + "Cannot use --follow and --tail together. Use --follow to stream logs or --tail to show recent logs." + ) + api = get_hf_api(token=token) try: logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7792ae08587..bff10ea2e13 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2020,3 +2020,9 @@ def test_logs_tail_short_flag(self, runner: CliRunner) -> None: assert "line 1" not in result.output assert "line 2" not in result.output assert "line 3" in result.output + + def test_logs_follow_and_tail_error(self, runner: CliRunner) -> None: + """Test that `hf jobs logs -f --tail 5 ` raises an error.""" + result = runner.invoke(app, ["jobs", "logs", "-f", "--tail", "5", "my-job-id"]) + assert result.exit_code != 0 + assert "Cannot use --follow and --tail together" in str(result.exception) From 89f6e1e32fd2b121c1a224932f4e2845ed87a454 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Mon, 9 Feb 2026 10:45:14 +0000 Subject: [PATCH 5/5] Move `follow` parameter into `_fetch_running_job_sse` Instead of a separate `_fetch_job_logs_no_follow` helper, integrate the follow/no-follow behavior directly into `_fetch_running_job_sse`. When `follow=False`, disables retries and breaks on timeout after collecting buffered events. Co-Authored-By: Claude Opus 4.6 --- src/huggingface_hub/hf_api.py | 66 +++++++---------------------------- 1 file changed, 13 insertions(+), 53 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index aad567de6eb..8b7625680f5 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -10263,6 +10263,7 @@ def _fetch_running_job_sse( timeout: int, skip_previous_events_on_retry: bool, double_check_job_has_finished_on_status_code_or_error: tuple[Union[int, Type[Exception]], ...], + follow: bool = True, namespace: Optional[str] = None, token: Union[bool, str, None] = None, ) -> Iterable[dict[str, Any]]: @@ -10270,7 +10271,7 @@ def _fetch_running_job_sse( namespace = self.whoami(token=token)["name"] # We don't use http_backoff since we need to check ourselves if the job is still running nb_tries = 0 - max_retries = 5 + max_retries = 5 if follow else 0 min_wait_time = 1 max_wait_time = 10 sleep_time = 0 @@ -10315,7 +10316,9 @@ def _fetch_running_job_sse( and isinstance(getattr(err.__context__, "__cause__", None), TimeoutError) ) if is_no_new_line_timeout: - # job is likely finished + if not follow: + break # no-follow mode: got all buffered events + # follow mode: job is likely finished pass elif type(err) in double_check_job_has_finished_on_status_code_or_error: pass @@ -10353,9 +10356,8 @@ def fetch_job_logs( The namespace where the Job is running. Defaults to the current user's namespace. follow (`bool`, *optional*): - If `True` (default), stream logs in real-time until the job completes (blocking). - If `False`, fetch only the currently available logs and return immediately (non-blocking). - Use `follow=False` when calling from scripts, agents, or CI pipelines that should not block. + If `True`, stream logs in real-time until the job completes (blocking). + If `False` (default), fetch only the currently available logs and return immediately (non-blocking). token `(Union[bool, str, None]`, *optional*): A valid user access token. If not provided, the locally saved token will be used, which is the @@ -10376,10 +10378,6 @@ def fetch_job_logs( ... print(log) ``` """ - if not follow: - yield from self._fetch_job_logs_no_follow(job_id=job_id, namespace=namespace, token=token) - return - # - We need to retry because sometimes the /logs doesn't return logs when the job just started. # (for example it can return only two lines: one for "Job started" and one empty line) # - Timeouts can happen in case of build errors @@ -10389,12 +10387,17 @@ def fetch_job_logs( # - there is a ": keep-alive" every 30 seconds seconds_between_keep_alive = 30 + # When not following, use a short timeout: the server replays historical logs + # quickly, then pauses waiting for new events (~30s keep-alive). 5 seconds is + # enough to receive all buffered logs. + timeout = 4 * seconds_between_keep_alive if follow else 5 for event in self._fetch_running_job_sse( job_id=job_id, route="logs", - timeout=4 * seconds_between_keep_alive, + timeout=timeout, skip_previous_events_on_retry=True, double_check_job_has_finished_on_status_code_or_error=tuple(), + follow=follow, namespace=namespace, token=token, ): @@ -10403,49 +10406,6 @@ def fetch_job_logs( log = event["data"] yield log - def _fetch_job_logs_no_follow( - self, - *, - job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Iterable[str]: - """Fetch only currently available logs and return immediately (non-blocking). - - Connects to the SSE log stream with a short read timeout. The server replays - all historical log lines quickly, then pauses waiting for new events (with - keep-alive every ~30s). When the short timeout fires without new data, we - treat that as "all buffered logs received" and return gracefully. - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - - # Short timeout: the server replays historical logs quickly, then pauses. - # 5 seconds is enough to receive all buffered logs. - read_timeout = 5 - try: - with get_session().stream( - "GET", - f"{self.endpoint}/api/jobs/{namespace}/{job_id}/logs", - headers=self._build_hf_headers(token=token), - timeout=read_timeout, - ) as response: - hf_raise_for_status(response) - for line in response.iter_lines(): - if line and line.startswith("data: {"): - event = json.loads(line[len("data: ") :]) - if not event["data"].startswith("===== Job started"): - yield event["data"] - except (httpx.ReadTimeout, httpcore.ReadTimeout): - # ReadTimeout means we received all buffered logs and the server is waiting - # for new events - this is the expected exit path for no-follow mode. - pass - except httpx.NetworkError as err: - # NetworkError can wrap a TimeoutError from the underlying socket. - is_timeout = err.__context__ and isinstance(getattr(err.__context__, "__cause__", None), TimeoutError) - if not is_timeout: - raise - def fetch_job_metrics( self, *,