Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions docs/source/en/package_reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -1116,7 +1116,7 @@ $ hf jobs [OPTIONS] COMMAND [ARGS]...
* `cancel`: Cancel a Job
* `hardware`: List available hardware options for Jobs
* `inspect`: Display detailed information on one or...
* `logs`: Fetch the logs of a Job
* `logs`: Fetch the logs of a Job.
* `ps`: List Jobs.
* `run`: Run a Job.
* `scheduled`: Create and manage scheduled Jobs on the Hub.
Expand Down Expand Up @@ -1203,7 +1203,10 @@ Learn more

### `hf jobs logs`

Fetch the logs of a Job
Fetch the logs of a Job.

By default, prints currently available logs and exits (non-blocking).
Use --follow/-f to stream logs in real-time until the job completes.

**Usage**:

Expand All @@ -1217,12 +1220,16 @@ $ hf jobs logs [OPTIONS] JOB_ID

**Options**:

* `-f, --follow`: Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.
* `-n, --tail INTEGER`: Number of lines to show from the end of the logs.
* `--namespace TEXT`: The namespace where the job will be running. Defaults to the current user's namespace.
* `--token TEXT`: A User Access Token generated from https://huggingface.co/settings/tokens.
* `--help`: Show this message and exit.

Examples
$ hf jobs logs <job_id>
$ hf jobs logs -f <job_id>
$ hf jobs logs --tail 20 <job_id>

Learn more
Use `hf <command> --help` for more information about a command.
Expand Down
46 changes: 40 additions & 6 deletions src/huggingface_hub/cli/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@
# List running or completed jobs
hf jobs ps [-a] [-f key=value] [--format TEMPLATE]

# Stream logs from a job
# Print logs from a job (non-blocking)
hf jobs logs <job-id>

# Stream logs from a job (blocking, like `docker logs -f`)
hf jobs logs -f <job-id>

# Stream resources usage stats and metrics from a job
hf jobs stats <job-id>

Expand Down Expand Up @@ -64,6 +67,7 @@
import os
import shutil
import time
from collections import deque
from dataclasses import asdict
from fnmatch import fnmatch
from pathlib import Path
Expand Down Expand Up @@ -312,20 +316,50 @@ def jobs_run(
if detach:
return
# Now let's stream the logs
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name):
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
print(log)


@jobs_cli.command("logs", examples=["hf jobs logs <job_id>"])
@jobs_cli.command(
"logs", examples=["hf jobs logs <job_id>", "hf jobs logs -f <job_id>", "hf jobs logs --tail 20 <job_id>"]
)
def jobs_logs(
job_id: JobIdArg,
follow: Annotated[
bool,
typer.Option(
"-f",
"--follow",
help="Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.",
),
] = False,
tail: Annotated[
Optional[int],
typer.Option(
"-n",
"--tail",
help="Number of lines to show from the end of the logs.",
),
] = None,
namespace: NamespaceOpt = None,
token: TokenOpt = None,
) -> None:
"""Fetch the logs of a Job"""
"""Fetch the logs of a Job.

By default, prints currently available logs and exits (non-blocking).
Use --follow/-f to stream logs in real-time until the job completes.
"""
if follow and tail is not None:
raise CLIError(
"Cannot use --follow and --tail together. Use --follow to stream logs or --tail to show recent logs."
)

api = get_hf_api(token=token)
try:
for log in api.fetch_job_logs(job_id=job_id, namespace=namespace):
logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow)
if tail is not None:
logs = deque(logs, maxlen=tail)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe raise a valueerror if both--follow and --tail are used? (they won't work together, right?)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added this + a test. test might be overkill but maybe good to catch changes to follow etc defaults later.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but maybe good to catch changes to follow etc defaults later.

Yep 👍 Thanks for adding it

for log in logs:
print(log)
except HfHubHTTPError as e:
status = e.response.status_code if e.response is not None else None
Expand Down Expand Up @@ -729,7 +763,7 @@ def jobs_uv_run(
if detach:
return
# Now let's stream the logs
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name):
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
print(log)


Expand Down
23 changes: 20 additions & 3 deletions src/huggingface_hub/hf_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -10263,14 +10263,15 @@ def _fetch_running_job_sse(
timeout: int,
skip_previous_events_on_retry: bool,
double_check_job_has_finished_on_status_code_or_error: tuple[Union[int, Type[Exception]], ...],
follow: bool = True,
namespace: Optional[str] = None,
token: Union[bool, str, None] = None,
) -> Iterable[dict[str, Any]]:
if namespace is None:
namespace = self.whoami(token=token)["name"]
# We don't use http_backoff since we need to check ourselves if the job is still running
nb_tries = 0
max_retries = 5
max_retries = 5 if follow else 0
min_wait_time = 1
max_wait_time = 10
sleep_time = 0
Expand Down Expand Up @@ -10315,7 +10316,9 @@ def _fetch_running_job_sse(
and isinstance(getattr(err.__context__, "__cause__", None), TimeoutError)
)
if is_no_new_line_timeout:
# job is likely finished
if not follow:
break # no-follow mode: got all buffered events
# follow mode: job is likely finished
pass
elif type(err) in double_check_job_has_finished_on_status_code_or_error:
pass
Expand All @@ -10339,6 +10342,7 @@ def fetch_job_logs(
*,
job_id: str,
namespace: Optional[str] = None,
follow: bool = False,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a breaking change no? we should probably set the default value to True and only the CLI should default to False

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

didn't see @Wauplin's comment. ok to break this then 👍

token: Union[bool, str, None] = None,
) -> Iterable[str]:
"""
Expand All @@ -10351,6 +10355,10 @@ def fetch_job_logs(
namespace (`str`, *optional*):
The namespace where the Job is running. Defaults to the current user's namespace.

follow (`bool`, *optional*):
If `True`, stream logs in real-time until the job completes (blocking).
If `False` (default), fetch only the currently available logs and return immediately (non-blocking).

token `(Union[bool, str, None]`, *optional*):
A valid user access token. If not provided, the locally saved token will be used, which is the
recommended authentication method. Set to `False` to disable authentication.
Expand All @@ -10364,6 +10372,10 @@ def fetch_job_logs(
>>> for log in fetch_job_logs(job_id=job.id):
... print(log)
Hello from HF compute!

>>> # Non-blocking: fetch only currently available logs
>>> for log in fetch_job_logs(job_id=job.id, follow=False):
... print(log)
```
"""
# - We need to retry because sometimes the /logs doesn't return logs when the job just started.
Expand All @@ -10375,12 +10387,17 @@ def fetch_job_logs(
# - there is a ": keep-alive" every 30 seconds

seconds_between_keep_alive = 30
# When not following, use a short timeout: the server replays historical logs
# quickly, then pauses waiting for new events (~30s keep-alive). 5 seconds is
# enough to receive all buffered logs.
timeout = 4 * seconds_between_keep_alive if follow else 5
for event in self._fetch_running_job_sse(
job_id=job_id,
route="logs",
timeout=4 * seconds_between_keep_alive,
timeout=timeout,
skip_previous_events_on_retry=True,
double_check_job_has_finished_on_status_code_or_error=tuple(),
follow=follow,
namespace=namespace,
token=token,
):
Expand Down
61 changes: 60 additions & 1 deletion tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1966,4 +1966,63 @@ def test_run_fetches_logs_with_correct_namespace(self, runner: CliRunner) -> Non
api.fetch_job_logs.return_value = iter(["log line 1"])
result = runner.invoke(app, ["jobs", "run", "ubuntu", "echo", "hello"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace="my-username")
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace="my-username", follow=True)

def test_logs_default_no_follow(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs <id>` defaults to follow=False (non-blocking, like `docker logs`)."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["line 1", "line 2"])
result = runner.invoke(app, ["jobs", "logs", "my-job-id"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=False)
assert "line 1" in result.output
assert "line 2" in result.output

def test_logs_follow_flag(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs -f <id>` passes follow=True."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["streaming line"])
result = runner.invoke(app, ["jobs", "logs", "-f", "my-job-id"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=True)
assert "streaming line" in result.output

def test_logs_follow_long_flag(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs --follow <id>` passes follow=True."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["streaming line"])
result = runner.invoke(app, ["jobs", "logs", "--follow", "my-job-id"])
assert result.exit_code == 0
api.fetch_job_logs.assert_called_once_with(job_id="my-job-id", namespace=None, follow=True)

def test_logs_tail(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs --tail 2 <id>` shows only the last 2 lines."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["line 1", "line 2", "line 3", "line 4"])
result = runner.invoke(app, ["jobs", "logs", "--tail", "2", "my-job-id"])
assert result.exit_code == 0
assert "line 1" not in result.output
assert "line 2" not in result.output
assert "line 3" in result.output
assert "line 4" in result.output

def test_logs_tail_short_flag(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs -n 1 <id>` shows only the last line."""
with patch("huggingface_hub.cli.jobs.get_hf_api") as api_cls:
api = api_cls.return_value
api.fetch_job_logs.return_value = iter(["line 1", "line 2", "line 3"])
result = runner.invoke(app, ["jobs", "logs", "-n", "1", "my-job-id"])
assert result.exit_code == 0
assert "line 1" not in result.output
assert "line 2" not in result.output
assert "line 3" in result.output

def test_logs_follow_and_tail_error(self, runner: CliRunner) -> None:
"""Test that `hf jobs logs -f --tail 5 <id>` raises an error."""
result = runner.invoke(app, ["jobs", "logs", "-f", "--tail", "5", "my-job-id"])
assert result.exit_code != 0
assert "Cannot use --follow and --tail together" in str(result.exception)
Loading