Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions docs/source/en/package_reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -1116,7 +1116,7 @@ $ hf jobs [OPTIONS] COMMAND [ARGS]...
* `cancel`: Cancel a Job
* `hardware`: List available hardware options for Jobs
* `inspect`: Display detailed information on one or...
* `logs`: Fetch the logs of a Job
* `logs`: Fetch the logs of a Job.
* `ps`: List Jobs.
* `run`: Run a Job.
* `scheduled`: Create and manage scheduled Jobs on the Hub.
Expand Down Expand Up @@ -1203,7 +1203,10 @@ Learn more

### `hf jobs logs`

Fetch the logs of a Job
Fetch the logs of a Job.

By default, prints currently available logs and exits (non-blocking).
Use --follow/-f to stream logs in real-time until the job completes.

**Usage**:

Expand All @@ -1217,12 +1220,16 @@ $ hf jobs logs [OPTIONS] JOB_ID

**Options**:

* `-f, --follow`: Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.
* `-n, --tail INTEGER`: Number of lines to show from the end of the logs.
* `--namespace TEXT`: The namespace where the job will be running. Defaults to the current user's namespace.
* `--token TEXT`: A User Access Token generated from https://huggingface.co/settings/tokens.
* `--help`: Show this message and exit.

Examples
$ hf jobs logs <job_id>
$ hf jobs logs -f <job_id>
$ hf jobs logs --tail 20 <job_id>

Learn more
Use `hf <command> --help` for more information about a command.
Expand Down
42 changes: 36 additions & 6 deletions src/huggingface_hub/cli/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@
# List running or completed jobs
hf jobs ps [-a] [-f key=value] [--format TEMPLATE]

# Stream logs from a job
# Print logs from a job (non-blocking)
hf jobs logs <job-id>

# Stream logs from a job (blocking, like `docker logs -f`)
hf jobs logs -f <job-id>

# Stream resources usage stats and metrics from a job
hf jobs stats <job-id>

Expand Down Expand Up @@ -312,20 +315,47 @@ def jobs_run(
if detach:
return
# Now let's stream the logs
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name):
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
print(log)


@jobs_cli.command("logs", examples=["hf jobs logs <job_id>"])
@jobs_cli.command(
"logs", examples=["hf jobs logs <job_id>", "hf jobs logs -f <job_id>", "hf jobs logs --tail 20 <job_id>"]
)
def jobs_logs(
job_id: JobIdArg,
follow: Annotated[
bool,
typer.Option(
"-f",
"--follow",
help="Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.",
),
] = False,
tail: Annotated[
Optional[int],
typer.Option(
"-n",
"--tail",
help="Number of lines to show from the end of the logs.",
),
] = None,
namespace: NamespaceOpt = None,
token: TokenOpt = None,
) -> None:
"""Fetch the logs of a Job"""
"""Fetch the logs of a Job.

By default, prints currently available logs and exits (non-blocking).
Use --follow/-f to stream logs in real-time until the job completes.
"""
api = get_hf_api(token=token)
try:
for log in api.fetch_job_logs(job_id=job_id, namespace=namespace):
logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow)
if tail is not None:
from collections import deque
Comment thread
davanstrien marked this conversation as resolved.
Resolved

logs = deque(logs, maxlen=tail)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe raise a valueerror if both--follow and --tail are used? (they won't work together, right?)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added this + a test. test might be overkill but maybe good to catch changes to follow etc defaults later.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but maybe good to catch changes to follow etc defaults later.

Yep 👍 Thanks for adding it

for log in logs:
print(log)
except HfHubHTTPError as e:
status = e.response.status_code if e.response is not None else None
Expand Down Expand Up @@ -729,7 +759,7 @@ def jobs_uv_run(
if detach:
return
# Now let's stream the logs
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name):
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
print(log)


Expand Down
57 changes: 57 additions & 0 deletions src/huggingface_hub/hf_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -10335,6 +10335,7 @@ def fetch_job_logs(
*,
job_id: str,
namespace: Optional[str] = None,
follow: bool = False,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a breaking change no? we should probably set the default value to True and only the CLI should default to False

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

didn't see @Wauplin's comment. ok to break this then 👍

token: Union[bool, str, None] = None,
) -> Iterable[str]:
"""
Expand All @@ -10347,6 +10348,11 @@ def fetch_job_logs(
namespace (`str`, *optional*):
The namespace where the Job is running. Defaults to the current user's namespace.

follow (`bool`, *optional*):
If `True` (default), stream logs in real-time until the job completes (blocking).
If `False`, fetch only the currently available logs and return immediately (non-blocking).
Use `follow=False` when calling from scripts, agents, or CI pipelines that should not block.

token `(Union[bool, str, None]`, *optional*):
A valid user access token. If not provided, the locally saved token will be used, which is the
recommended authentication method. Set to `False` to disable authentication.
Expand All @@ -10360,8 +10366,16 @@ def fetch_job_logs(
>>> for log in fetch_job_logs(job_id=job.id):
... print(log)
Hello from HF compute!

>>> # Non-blocking: fetch only currently available logs
>>> for log in fetch_job_logs(job_id=job.id, follow=False):
... print(log)
```
"""
if not follow:
yield from self._fetch_job_logs_no_follow(job_id=job_id, namespace=namespace, token=token)
return

# - We need to retry because sometimes the /logs doesn't return logs when the job just started.
# (for example it can return only two lines: one for "Job started" and one empty line)
# - Timeouts can happen in case of build errors
Expand All @@ -10385,6 +10399,49 @@ def fetch_job_logs(
log = event["data"]
yield log

def _fetch_job_logs_no_follow(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

instead of having a new helper, could we define add a follow parameter in _fetch_running_job_sse ?

(if not possible that's fine)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Claude attempted a refactor of this. Don't think it looks too gnarly but let me know if you think the branching in _fetch_running_job_sse looks too much now.

self,
*,
job_id: str,
namespace: Optional[str] = None,
token: Union[bool, str, None] = None,
) -> Iterable[str]:
"""Fetch only currently available logs and return immediately (non-blocking).

Connects to the SSE log stream with a short read timeout. The server replays
all historical log lines quickly, then pauses waiting for new events (with
keep-alive every ~30s). When the short timeout fires without new data, we
treat that as "all buffered logs received" and return gracefully.
"""
if namespace is None:
namespace = self.whoami(token=token)["name"]

# Short timeout: the server replays historical logs quickly, then pauses.
# 5 seconds is enough to receive all buffered logs.
read_timeout = 5

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This may not be a good assumption by Claude

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel that it's fine (smaller might miss some logs on slow environments, higher would degrade UX)

try:
with get_session().stream(
"GET",
f"{self.endpoint}/api/jobs/{namespace}/{job_id}/logs",
headers=self._build_hf_headers(token=token),
timeout=read_timeout,
) as response:
hf_raise_for_status(response)
for line in response.iter_lines():
if line and line.startswith("data: {"):
event = json.loads(line[len("data: ") :])
if not event["data"].startswith("===== Job started"):
yield event["data"]
except (httpx.ReadTimeout, httpcore.ReadTimeout):
# ReadTimeout means we received all buffered logs and the server is waiting
# for new events - this is the expected exit path for no-follow mode.
pass
except httpx.NetworkError as err:
# NetworkError can wrap a TimeoutError from the underlying socket.
is_timeout = err.__context__ and isinstance(getattr(err.__context__, "__cause__", None), TimeoutError)
if not is_timeout:
raise

def fetch_job_metrics(
self,
*,
Expand Down
55 changes: 54 additions & 1 deletion tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1966,4 +1966,57 @@ def test_run_fetches_logs_with_correct_namespace(self, runner: CliRunner) -> Non
api.fetch_job_logs.return_value = iter(["log line 1"])
result = runner.invoke(app, ["jobs", "run", "ubuntu", "echo", "hello"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace="my-username")
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace="my-username", follow=True)

def test_logs_default_no_follow(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs <id>` defaults to follow=False (non-blocking, like `docker logs`)."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["line 1", "line 2"])
result = runner.invoke(app, ["jobs", "logs", "my-job-id"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=False)
assert "line 1" in result.output
assert "line 2" in result.output

def test_logs_follow_flag(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs -f <id>` passes follow=True."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["streaming line"])
result = runner.invoke(app, ["jobs", "logs", "-f", "my-job-id"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=True)
assert "streaming line" in result.output

def test_logs_follow_long_flag(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs --follow <id>` passes follow=True."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["streaming line"])
result = runner.invoke(app, ["jobs", "logs", "--follow", "my-job-id"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=True)

def test_logs_tail(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs --tail 2 <id>` shows only the last 2 lines."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["line 1", "line 2", "line 3", "line 4"])
result = runner.invoke(app, ["jobs", "logs", "--tail", "2", "my-job-id"])
assert result.exit_code == 0
assert "line 1" not in result.output
assert "line 2" not in result.output
assert "line 3" in result.output
assert "line 4" in result.output

def test_logs_tail_short_flag(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs -n 1 <id>` shows only the last line."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["line 1", "line 2", "line 3"])
result = runner.invoke(app, ["jobs", "logs", "-n", "1", "my-job-id"])
assert result.exit_code == 0
assert "line 1" not in result.output
assert "line 2" not in result.output
assert "line 3" in result.output
Loading