Repository navigation
Make hf jobs logs agent-friendly with --follow/-f and --tail/-n flags
#3783
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
0707ad9
97f49fa
305b5cf
3af4150
80c38c7
89f6e1e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
--follow/-f and --tail/-n flags to hf jobs logs
`hf jobs logs` currently streams logs indefinitely via SSE until a job completes, blocking the caller. This makes it harder to use for agents, CI scripts, and any programmatic consumer that needs to check logs without hanging. This follows Docker's convention: `docker logs` prints existing logs and exits by default, while `docker logs -f` opts into streaming. Similarly: - `hf jobs logs <id>` now prints available logs and exits (non-blocking) - `hf jobs logs -f <id>` streams until the job completes (blocking) - `hf jobs logs --tail N <id>` shows only the last N lines The Python API adds a `follow` parameter to `fetch_job_logs()` (default `True` for backward compatibility). The CLI defaults to `follow=False`. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,9 +20,12 @@ | |
| # List running or completed jobs | ||
| hf jobs ps [-a] [-f key=value] [--format TEMPLATE] | ||
|
|
||
| # Stream logs from a job | ||
| # Print logs from a job (non-blocking) | ||
| hf jobs logs <job-id> | ||
|
|
||
| # Stream logs from a job (blocking, like `docker logs -f`) | ||
| hf jobs logs -f <job-id> | ||
|
|
||
| # Stream resources usage stats and metrics from a job | ||
| hf jobs stats <job-id> | ||
|
|
||
|
|
@@ -312,20 +315,47 @@ def jobs_run( | |
| if detach: | ||
| return | ||
| # Now let's stream the logs | ||
| for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name): | ||
| for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): | ||
| print(log) | ||
|
|
||
|
|
||
| @jobs_cli.command("logs", examples=["hf jobs logs <job_id>"]) | ||
| @jobs_cli.command( | ||
| "logs", examples=["hf jobs logs <job_id>", "hf jobs logs -f <job_id>", "hf jobs logs --tail 20 <job_id>"] | ||
| ) | ||
| def jobs_logs( | ||
| job_id: JobIdArg, | ||
| follow: Annotated[ | ||
| bool, | ||
| typer.Option( | ||
| "-f", | ||
| "--follow", | ||
| help="Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.", | ||
| ), | ||
| ] = False, | ||
| tail: Annotated[ | ||
| Optional[int], | ||
| typer.Option( | ||
| "-n", | ||
| "--tail", | ||
| help="Number of lines to show from the end of the logs.", | ||
| ), | ||
| ] = None, | ||
| namespace: NamespaceOpt = None, | ||
| token: TokenOpt = None, | ||
| ) -> None: | ||
| """Fetch the logs of a Job""" | ||
| """Fetch the logs of a Job. | ||
|
|
||
| By default, prints currently available logs and exits (non-blocking). | ||
| Use --follow/-f to stream logs in real-time until the job completes. | ||
| """ | ||
| api = get_hf_api(token=token) | ||
| try: | ||
| for log in api.fetch_job_logs(job_id=job_id, namespace=namespace): | ||
| logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow) | ||
| if tail is not None: | ||
| from collections import deque | ||
|
|
||
| logs = deque(logs, maxlen=tail) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe raise a valueerror if both
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added this + a test. test might be overkill but maybe good to catch changes to follow etc defaults later.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Yep 👍 Thanks for adding it |
||
| for log in logs: | ||
| print(log) | ||
| except HfHubHTTPError as e: | ||
| status = e.response.status_code if e.response is not None else None | ||
|
|
@@ -729,7 +759,7 @@ def jobs_uv_run( | |
| if detach: | ||
| return | ||
| # Now let's stream the logs | ||
| for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name): | ||
| for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): | ||
| print(log) | ||
|
|
||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10335,6 +10335,7 @@ def fetch_job_logs( | |
| *, | ||
| job_id: str, | ||
| namespace: Optional[str] = None, | ||
| follow: bool = True, | ||
| token: Union[bool, str, None] = None, | ||
| ) -> Iterable[str]: | ||
| """ | ||
|
|
@@ -10347,6 +10348,11 @@ def fetch_job_logs( | |
| namespace (`str`, *optional*): | ||
| The namespace where the Job is running. Defaults to the current user's namespace. | ||
|
|
||
| follow (`bool`, *optional*): | ||
| If `True` (default), stream logs in real-time until the job completes (blocking). | ||
| If `False`, fetch only the currently available logs and return immediately (non-blocking). | ||
| Use `follow=False` when calling from scripts, agents, or CI pipelines that should not block. | ||
|
|
||
| token `(Union[bool, str, None]`, *optional*): | ||
| A valid user access token. If not provided, the locally saved token will be used, which is the | ||
| recommended authentication method. Set to `False` to disable authentication. | ||
|
|
@@ -10360,8 +10366,16 @@ def fetch_job_logs( | |
| >>> for log in fetch_job_logs(job_id=job.id): | ||
| ... print(log) | ||
| Hello from HF compute! | ||
|
|
||
| >>> # Non-blocking: fetch only currently available logs | ||
| >>> for log in fetch_job_logs(job_id=job.id, follow=False): | ||
| ... print(log) | ||
| ``` | ||
| """ | ||
| if not follow: | ||
| yield from self._fetch_job_logs_no_follow(job_id=job_id, namespace=namespace, token=token) | ||
| return | ||
|
|
||
| # - We need to retry because sometimes the /logs doesn't return logs when the job just started. | ||
| # (for example it can return only two lines: one for "Job started" and one empty line) | ||
| # - Timeouts can happen in case of build errors | ||
|
|
@@ -10385,6 +10399,49 @@ def fetch_job_logs( | |
| log = event["data"] | ||
| yield log | ||
|
|
||
| def _fetch_job_logs_no_follow( | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. instead of having a new helper, could we define add a (if not possible that's fine)
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Claude attempted a refactor of this. Don't think it looks too gnarly but let me know if you think the branching in |
||
| self, | ||
| *, | ||
| job_id: str, | ||
| namespace: Optional[str] = None, | ||
| token: Union[bool, str, None] = None, | ||
| ) -> Iterable[str]: | ||
| """Fetch only currently available logs and return immediately (non-blocking). | ||
|
|
||
| Connects to the SSE log stream with a short read timeout. The server replays | ||
| all historical log lines quickly, then pauses waiting for new events (with | ||
| keep-alive every ~30s). When the short timeout fires without new data, we | ||
| treat that as "all buffered logs received" and return gracefully. | ||
| """ | ||
| if namespace is None: | ||
| namespace = self.whoami(token=token)["name"] | ||
|
|
||
| # Short timeout: the server replays historical logs quickly, then pauses. | ||
| # 5 seconds is enough to receive all buffered logs. | ||
| read_timeout = 5 | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This may not be a good assumption by Claude
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I feel that it's fine (smaller might miss some logs on slow environments, higher would degrade UX) |
||
| try: | ||
| with get_session().stream( | ||
| "GET", | ||
| f"{self.endpoint}/api/jobs/{namespace}/{job_id}/logs", | ||
| headers=self._build_hf_headers(token=token), | ||
| timeout=read_timeout, | ||
| ) as response: | ||
| hf_raise_for_status(response) | ||
| for line in response.iter_lines(): | ||
| if line and line.startswith("data: {"): | ||
| event = json.loads(line[len("data: ") :]) | ||
| if not event["data"].startswith("===== Job started"): | ||
| yield event["data"] | ||
| except (httpx.ReadTimeout, httpcore.ReadTimeout): | ||
| # ReadTimeout means we received all buffered logs and the server is waiting | ||
| # for new events - this is the expected exit path for no-follow mode. | ||
| pass | ||
| except httpx.NetworkError as err: | ||
| # NetworkError can wrap a TimeoutError from the underlying socket. | ||
| is_timeout = err.__context__ and isinstance(getattr(err.__context__, "__cause__", None), TimeoutError) | ||
| if not is_timeout: | ||
| raise | ||
|
|
||
| def fetch_job_metrics( | ||
| self, | ||
| *, | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.