From ef7f63b166d6ea8c58cd8d238edc932d69bae631 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Tue, 29 Sep 2026 16:33:31 +0200 Subject: [PATCH 1/2] [Repos] Report background file copy of duplicated repos The Hub now copies LFS/Xet files of a duplicated repo in a background job. `duplicate_repo` returns a `DuplicatedRepoUrl` with `files_copy_pending`, `get_duplication_status` returns the copy progress, and the CLI warns when the copy is still pending and adds `hf repos duplication-status`. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/source/en/guides/cli.md | 24 ++++- docs/source/en/guides/repository.md | 11 ++ docs/source/en/package_reference/cli.md | 29 ++++++ docs/source/en/package_reference/hf_api.md | 8 ++ src/huggingface_hub/__init__.py | 9 ++ src/huggingface_hub/cli/repos.py | 23 +++++ src/huggingface_hub/hf_api.py | 114 ++++++++++++++++++++- tests/test_cli.py | 27 ++++- tests/test_hf_api.py | 21 ++++ 9 files changed, 258 insertions(+), 8 deletions(-) diff --git a/docs/source/en/guides/cli.md b/docs/source/en/guides/cli.md index 9e0a619b150..f1c7347e317 100644 --- a/docs/source/en/guides/cli.md +++ b/docs/source/en/guides/cli.md @@ -1368,7 +1368,7 @@ To view the diff of a pull request directly in your terminal, use `hf discussion ## hf repos -`hf repos` lets you list, create, delete, move repositories, update their settings, and delete files on the Hugging Face Hub. It also includes subcommands to manage branches and tags. +`hf repos` lets you list, create, duplicate, delete, move repositories, update their settings, and delete files on the Hugging Face Hub. It also includes subcommands to manage branches and tags. ### List repos @@ -1434,6 +1434,28 @@ Create a repo in a specific region: >>> hf repos create my-model --region us ``` +### Duplicate a repo + +Use `hf repos duplicate` to copy an existing model, dataset, or Space to your account (or to an organization). The copy happens on the server and preserves the git history: + +```bash +>>> hf repos duplicate openai/gdpval --type dataset +>>> hf repos duplicate openai/gdpval myorg/my-gdpval --type dataset --private +``` + +The LFS/Xet files of the new repo may still be copied in the background after the command returns. In that case, a warning is printed and downloading these files from the new repo may fail until the copy is complete. Use `hf repos duplication-status` to check the progress: + +```bash +>>> hf repos duplication-status myorg/my-gdpval --type dataset +{ + "pending": true, + "files_total": 42, + "files_copied": 10 +} +``` + +`"pending": false` means the copy is complete. + ### Delete a repo ```bash diff --git a/docs/source/en/guides/repository.md b/docs/source/en/guides/repository.md index a0fe27aaa13..e5f4e7f3535 100644 --- a/docs/source/en/guides/repository.md +++ b/docs/source/en/guides/repository.md @@ -155,6 +155,17 @@ RepoUrl('https://huggingface.co/spaces/nateraw/dreambooth-training',...) RepoUrl('https://huggingface.co/datasets/nateraw/gdpval',...) ``` +The git history is copied right away, but the LFS/Xet files may still be copied in the background once [`duplicate_repo`] returns. Until the copy is complete, downloading these files from the new repo may fail. Check `files_copy_pending` on the returned value and use [`get_duplication_status`] to follow the progress: + +```py +>>> from huggingface_hub import duplicate_repo, get_duplication_status +>>> repo_url = duplicate_repo("openai/gdpval", repo_type="dataset") +>>> repo_url.files_copy_pending +True +>>> get_duplication_status(repo_url.repo_id, repo_type="dataset") +DuplicationStatus(pending=True, files_total=42, files_copied=10) +``` + ## Search for Spaces The Hub provides a semantic search API for discovering Spaces. You can search using natural language queries with [`search_spaces`]: diff --git a/docs/source/en/package_reference/cli.md b/docs/source/en/package_reference/cli.md index 183e3acfa6f..ce93fcf7140 100644 --- a/docs/source/en/package_reference/cli.md +++ b/docs/source/en/package_reference/cli.md @@ -3494,6 +3494,7 @@ $ hf repos [OPTIONS] COMMAND [ARGS]... * `delete`: Delete a repo from the Hub. * `delete-files`: Delete files from a repo on the Hub. * `duplicate`: Duplicate a repo on the Hub (model, dataset, or Space). +* `duplication-status`: Get the progress of the background file copy of a duplicated repo. * `list`: List all repos (models, datasets, spaces, buckets) with storage info. [alias: ls] * `move`: Move a repository from a namespace to another namespace. * `settings`: Update the settings of a repository. @@ -3776,6 +3777,34 @@ Learn more Read the documentation at https://huggingface.co/docs/huggingface_hub/en/guides/cli +### `hf repos duplication-status` + +Get the progress of the background file copy of a duplicated repo. + +**Usage**: + +```console +$ hf repos duplication-status [OPTIONS] REPO_ID +``` + +**Arguments**: + +* `REPO_ID`: The ID of the repo (e.g. `username/repo-name` or `spaces/username/repo-name`). [required] + +**Options**: + +* `--type, --repo-type [model|dataset|space|kernel]`: The type of repository (model, dataset, space, or kernel). [default: model] +* `--token TEXT`: A User Access Token generated from https://huggingface.co/settings/tokens. +* `--help`: Show this message and exit. + +Examples + $ hf repos duplication-status myorg/my-gdpval --type dataset + +Learn more + Use `hf --help` for more information about a command. + Read the documentation at https://huggingface.co/docs/huggingface_hub/en/guides/cli + + ### `hf repos list` List all repos (models, datasets, spaces, buckets) with storage info. [alias: ls] diff --git a/docs/source/en/package_reference/hf_api.md b/docs/source/en/package_reference/hf_api.md index b237b090be1..e67f61ee795 100644 --- a/docs/source/en/package_reference/hf_api.md +++ b/docs/source/en/package_reference/hf_api.md @@ -81,6 +81,14 @@ models = hf_api.list_models() [[autodoc]] huggingface_hub.DryRunFileInfo +### DuplicatedRepoUrl + +[[autodoc]] huggingface_hub.hf_api.DuplicatedRepoUrl + +### DuplicationStatus + +[[autodoc]] huggingface_hub.hf_api.DuplicationStatus + ### GitRefInfo [[autodoc]] huggingface_hub.hf_api.GitRefInfo diff --git a/src/huggingface_hub/__init__.py b/src/huggingface_hub/__init__.py index 11b96365033..0b62bdc7458 100644 --- a/src/huggingface_hub/__init__.py +++ b/src/huggingface_hub/__init__.py @@ -196,6 +196,8 @@ "CommitInfo", "DatasetInfo", "DatasetLeaderboardEntry", + "DuplicatedRepoUrl", + "DuplicationStatus", "GitCommitInfo", "GitRefInfo", "GitRefs", @@ -271,6 +273,7 @@ "get_dataset_leaderboard", "get_dataset_tags", "get_discussion_details", + "get_duplication_status", "get_full_repo_name", "get_inference_endpoint", "get_local_safetensors_metadata", @@ -735,6 +738,8 @@ "DocumentQuestionAnsweringOutputElement", "DocumentQuestionAnsweringParameters", "DryRunFileInfo", + "DuplicatedRepoUrl", + "DuplicationStatus", "EvalResult", "EvalResultEntry", "FLAX_WEIGHTS_NAME", @@ -1014,6 +1019,7 @@ "get_dataset_leaderboard", "get_dataset_tags", "get_discussion_details", + "get_duplication_status", "get_full_repo_name", "get_hf_file_metadata", "get_inference_endpoint", @@ -1396,6 +1402,8 @@ def __dir__(): CommitInfo, # noqa: F401 DatasetInfo, # noqa: F401 DatasetLeaderboardEntry, # noqa: F401 + DuplicatedRepoUrl, # noqa: F401 + DuplicationStatus, # noqa: F401 GitCommitInfo, # noqa: F401 GitRefInfo, # noqa: F401 GitRefs, # noqa: F401 @@ -1471,6 +1479,7 @@ def __dir__(): get_dataset_leaderboard, # noqa: F401 get_dataset_tags, # noqa: F401 get_discussion_details, # noqa: F401 + get_duplication_status, # noqa: F401 get_full_repo_name, # noqa: F401 get_inference_endpoint, # noqa: F401 get_local_safetensors_metadata, # noqa: F401 diff --git a/src/huggingface_hub/cli/repos.py b/src/huggingface_hub/cli/repos.py index 3855ed8ea62..d7bcf28fa76 100644 --- a/src/huggingface_hub/cli/repos.py +++ b/src/huggingface_hub/cli/repos.py @@ -305,6 +305,29 @@ def repo_duplicate( space_volumes=parse_volumes(volume), ) out.result("Repo duplicated", from_id=from_id, to_id=repo_url.repo_id, url=str(repo_url)) + if repo_url.files_copy_pending: + type_flag = "" if repo_type == RepoType.model else f" --type {repo_type.value}" + out.warning( + f"LFS/Xet files are still being copied in the background. Downloading them from '{repo_url.repo_id}'" + " may fail until the copy is complete." + ) + out.hint(f"Use 'hf repos duplication-status {repo_url.repo_id}{type_flag}' to check the copy progress.") + + +@repos_cli.command( + "duplication-status", + examples=[ + "hf repos duplication-status myorg/my-gdpval --type dataset", + ], +) +def repo_duplication_status( + repo_id: RepoIdArg, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Get the progress of the background file copy of a duplicated repo.""" + api = get_hf_api(token=token) + out.dict(api.get_duplication_status(repo_id, repo_type=repo_type.value, token=token)) @repos_cli.command("delete", examples=["hf repos delete my-model"]) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index bd71b550aa2..09d5d10d1dc 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -568,6 +568,49 @@ def __repr__(self) -> str: return f"RepoUrl('{self}', endpoint='{self.endpoint}', repo_type='{self.repo_type}', repo_id='{self.repo_id}')" +class DuplicatedRepoUrl(RepoUrl): + """[`RepoUrl`] returned by [`duplicate_repo`], with information about the background file copy. + + Attributes: + files_copy_pending (`bool`): + `True` if the LFS/Xet files of the new repo are still being copied in the background. Until the copy is + complete, downloading these files may fail. Use [`get_duplication_status`] to check progress. + """ + + def __new__(cls, url: Any, endpoint: str | None = None, files_copy_pending: bool = False): + return super().__new__(cls, url, endpoint=endpoint) + + def __init__(self, url: Any, endpoint: str | None = None, files_copy_pending: bool = False) -> None: + super().__init__(url, endpoint=endpoint) + self.files_copy_pending = files_copy_pending + + +@dataclass +class DuplicationStatus: + """Progress of the background LFS/Xet file copy of a duplicated repo. + + Returned by [`get_duplication_status`]. + + Attributes: + pending (`bool`): + `True` while the LFS/Xet files are still being copied. `False` once the copy is complete (or if the repo + has no copy in progress). + files_total (`int`, *optional*): + Number of LFS/Xet files to copy. Only set while the copy is pending. + files_copied (`int`, *optional*): + Number of files copied so far. Only set while the copy is pending. + """ + + pending: bool + files_total: int | None + files_copied: int | None + + def __init__(self, **kwargs: Any) -> None: + self.pending = kwargs["pending"] + self.files_total = kwargs.get("filesTotal") + self.files_copied = kwargs.get("filesCopied") + + def _resolve_copy_target_path( src_file_path: str, src_root_path: str | None, @@ -8612,12 +8655,16 @@ def duplicate_repo( space_secrets: list[dict[str, str]] | None = None, space_variables: list[dict[str, str]] | None = None, space_volumes: list[Volume] | None = None, - ) -> RepoUrl: + ) -> DuplicatedRepoUrl: """Duplicate a repo on the Hub (model, dataset, or Space). This performs a server-side copy that preserves full git history and LFS objects without requiring a local download/upload round-trip. + The git history is copied right away, but the LFS/Xet files may still be copied in the background once this + method returns. In that case, `files_copy_pending` is `True` on the returned value and downloading these files + from the new repo may fail until the copy is complete. Use [`get_duplication_status`] to check progress. + Args: from_id (`str`): ID of the repo to duplicate. Example: `"openai/gdpval"`. @@ -8666,8 +8713,8 @@ def duplicate_repo( Only applicable if repo_type is "space". Returns: - [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing - attributes like `endpoint`, `repo_type` and `repo_id`. + [`DuplicatedRepoUrl`]: URL to the newly created repo. Value is a subclass of [`RepoUrl`] (and of `str`) + containing attributes like `endpoint`, `repo_type`, `repo_id` and `files_copy_pending`. Raises: [`~utils.RepositoryNotFoundError`]: @@ -8681,8 +8728,11 @@ def duplicate_repo( >>> from huggingface_hub import duplicate_repo # Duplicate a model to your account - >>> duplicate_repo("google/gemma-7b") + >>> repo_url = duplicate_repo("google/gemma-7b") + >>> repo_url RepoUrl('https://huggingface.co/nateraw/gemma-7b',...) + >>> repo_url.files_copy_pending + True # Duplicate a dataset with a custom name >>> duplicate_repo("openai/gdpval", to_id="myorg/my-gdpval", repo_type="dataset") @@ -8770,7 +8820,60 @@ def duplicate_repo( else: raise - return RepoUrl(r.json()["url"], endpoint=self.endpoint) + data = r.json() + return DuplicatedRepoUrl( + data["url"], endpoint=self.endpoint, files_copy_pending=data.get("filesCopyPending", False) + ) + + @validate_hf_hub_args + def get_duplication_status( + self, + repo_id: str, + *, + repo_type: str | None = None, + token: bool | str | None = None, + ) -> DuplicationStatus: + """Get the progress of the background LFS/Xet file copy of a repo created with [`duplicate_repo`]. + + When duplicating a repo, the LFS/Xet files are copied in the background. Until the copy is complete, + downloading these files from the new repo may fail. + + Args: + repo_id (`str`): + ID of the new repo (the duplicate, not the original one). Example: `"myorg/my-gdpval"`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if the repo is a dataset or a Space, `None` or `"model"` if it is a + model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`DuplicationStatus`]: The copy status. `pending` is `False` once the copy is complete (or if the repo + has no copy in progress). `files_total` and `files_copied` are only set while the copy is pending. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If the repo cannot be found. This may be because it doesn't exist, or because it is set to `private` + and you do not have access. + + Example: + ```python + >>> from huggingface_hub import duplicate_repo, get_duplication_status + >>> repo_url = duplicate_repo("openai/gdpval", repo_type="dataset") + >>> get_duplication_status(repo_url.repo_id, repo_type="dataset") + DuplicationStatus(pending=True, files_total=42, files_copied=10) + ``` + """ + repo_type = repo_type or constants.REPO_TYPE_MODEL + r = get_session().get( + f"{self.endpoint}/api/{repo_type}s/{repo_id}/duplicate/status", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(r) + return DuplicationStatus(**r.json()) @validate_hf_hub_args def set_space_volumes( @@ -15198,6 +15301,7 @@ def get_local_safetensors_metadata(path: str | Path) -> SafetensorsRepoMetadata: pause_space = api.pause_space restart_space = api.restart_space duplicate_repo = api.duplicate_repo +get_duplication_status = api.get_duplication_status set_space_volumes = api.set_space_volumes delete_space_volumes = api.delete_space_volumes enable_space_dev_mode = api.enable_space_dev_mode diff --git a/tests/test_cli.py b/tests/test_cli.py index f572f894649..920b078e0b7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -32,7 +32,7 @@ from huggingface_hub.cli.skills import build_skill_md from huggingface_hub.cli.upload import _resolve_upload_paths, upload from huggingface_hub.errors import CLIError, DeviceCodeError, HfUriError, RevisionNotFoundError -from huggingface_hub.hf_api import ModelInfo +from huggingface_hub.hf_api import DuplicationStatus, ModelInfo from huggingface_hub.utils import ( CachedFileInfo, CachedRepoInfo, @@ -1810,6 +1810,25 @@ def test_repo_duplicate_secret_from_env(self, runner: CliRunner) -> None: space_volumes=None, ) + def test_repo_duplicate_files_copy_pending(self, runner: CliRunner) -> None: + with patch("huggingface_hub.cli.repos.get_hf_api") as api_cls: + api_cls.return_value.duplicate_repo.return_value = Mock(repo_id="myorg/my-copy", files_copy_pending=True) + result = runner.invoke(app, ["repos", "duplicate", "owner/repo", "myorg/my-copy", "--type", "dataset"]) + assert result.exit_code == 0 + assert "still being copied in the background" in result.stderr + assert "hf repos duplication-status myorg/my-copy --type dataset" in result.stderr + + def test_repo_duplication_status(self, runner: CliRunner) -> None: + with patch("huggingface_hub.cli.repos.get_hf_api") as api_cls: + api = api_cls.return_value + api.get_duplication_status.return_value = DuplicationStatus(pending=True, filesTotal=42, filesCopied=10) + result = runner.invoke( + app, ["repos", "duplication-status", "myorg/my-copy", "--type", "dataset", "--format", "json"] + ) + assert result.exit_code == 0 + api.get_duplication_status.assert_called_once_with("myorg/my-copy", repo_type="dataset", token=None) + assert json.loads(result.stdout) == {"pending": True, "files_total": 42, "files_copied": 10} + class TestRepoMoveCommand: def test_repo_move_basic(self, runner: CliRunner) -> None: @@ -5297,7 +5316,11 @@ def test_prefix_on_duplicate_from_id(self, runner: CliRunner) -> None: api_cls.return_value.duplicate_repo.return_value = type( "RepoUrl", (), - {"repo_id": "user/my-space-copy", "__str__": lambda s: "https://hf.co/user/my-space-copy"}, + { + "repo_id": "user/my-space-copy", + "files_copy_pending": False, + "__str__": lambda s: "https://hf.co/user/my-space-copy", + }, )() result = runner.invoke(app, ["repos", "duplicate", "spaces/user/my-space"]) assert result.exit_code == 0, result.output diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index 08226f1fbe9..f7ce6b3f1ec 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -57,6 +57,7 @@ CommitInfo, DatasetInfo, DatasetLeaderboardEntry, + DuplicationStatus, ExpandDatasetProperty_T, ExpandModelProperty_T, ExpandSpaceProperty_T, @@ -4633,6 +4634,26 @@ def test_create_inference_endpoint_private_link_payload(mocker): assert "accountId" not in payload +def test_duplicate_repo_files_copy_pending(mocker): + mock_session = mocker.patch("huggingface_hub.hf_api.get_session").return_value + mock_session.post.return_value.json.return_value = { + "url": f"{ENDPOINT_STAGING}/datasets/myorg/my-copy", + "filesCopyPending": True, + } + mock_session.get.return_value.json.return_value = {"pending": True, "filesTotal": 42, "filesCopied": 10} + api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN) + + repo_url = api.duplicate_repo("owner/repo", "myorg/my-copy", repo_type="dataset") + assert isinstance(repo_url, RepoUrl) + assert repo_url.repo_id == "myorg/my-copy" + assert repo_url.files_copy_pending is True + + status = api.get_duplication_status("myorg/my-copy", repo_type="dataset") + assert status == DuplicationStatus(pending=True, filesTotal=42, filesCopied=10) + assert (status.pending, status.files_total, status.files_copied) == (True, 42, 10) + assert mock_session.get.call_args.args[0] == f"{ENDPOINT_STAGING}/api/datasets/myorg/my-copy/duplicate/status" + + @pytest.mark.parametrize( "custom_image, registry_credentials, match", [ From acf15f0a3e70cd56ffdc8b3363f6a000a2c35f05 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 30 Sep 2026 16:53:07 +0200 Subject: [PATCH 2/2] [Repos] Simplify duplicate copy status: drop status API/CLI and tests Keep only DuplicatedRepoUrl.files_copy_pending and the CLI warning. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/source/en/guides/cli.md | 13 +-- docs/source/en/guides/repository.md | 6 +- docs/source/en/package_reference/cli.md | 29 ------- docs/source/en/package_reference/hf_api.md | 4 - src/huggingface_hub/__init__.py | 6 -- src/huggingface_hub/cli/repos.py | 18 ---- src/huggingface_hub/hf_api.py | 96 ++-------------------- tests/test_cli.py | 21 +---- tests/test_hf_api.py | 21 ----- 9 files changed, 11 insertions(+), 203 deletions(-) diff --git a/docs/source/en/guides/cli.md b/docs/source/en/guides/cli.md index f1c7347e317..45313476e32 100644 --- a/docs/source/en/guides/cli.md +++ b/docs/source/en/guides/cli.md @@ -1443,18 +1443,7 @@ Use `hf repos duplicate` to copy an existing model, dataset, or Space to your ac >>> hf repos duplicate openai/gdpval myorg/my-gdpval --type dataset --private ``` -The LFS/Xet files of the new repo may still be copied in the background after the command returns. In that case, a warning is printed and downloading these files from the new repo may fail until the copy is complete. Use `hf repos duplication-status` to check the progress: - -```bash ->>> hf repos duplication-status myorg/my-gdpval --type dataset -{ - "pending": true, - "files_total": 42, - "files_copied": 10 -} -``` - -`"pending": false` means the copy is complete. +The LFS/Xet files of the new repo may still be copied in the background after the command returns. In that case, a warning is printed and downloading these files from the new repo may fail until the copy is complete. ### Delete a repo diff --git a/docs/source/en/guides/repository.md b/docs/source/en/guides/repository.md index e5f4e7f3535..2b0a8f30095 100644 --- a/docs/source/en/guides/repository.md +++ b/docs/source/en/guides/repository.md @@ -155,15 +155,13 @@ RepoUrl('https://huggingface.co/spaces/nateraw/dreambooth-training',...) RepoUrl('https://huggingface.co/datasets/nateraw/gdpval',...) ``` -The git history is copied right away, but the LFS/Xet files may still be copied in the background once [`duplicate_repo`] returns. Until the copy is complete, downloading these files from the new repo may fail. Check `files_copy_pending` on the returned value and use [`get_duplication_status`] to follow the progress: +The git history is copied right away, but the LFS/Xet files may still be copied in the background once [`duplicate_repo`] returns. Until the copy is complete, downloading these files from the new repo may fail. Check `files_copy_pending` on the returned value to know if the copy is still in progress: ```py ->>> from huggingface_hub import duplicate_repo, get_duplication_status +>>> from huggingface_hub import duplicate_repo >>> repo_url = duplicate_repo("openai/gdpval", repo_type="dataset") >>> repo_url.files_copy_pending True ->>> get_duplication_status(repo_url.repo_id, repo_type="dataset") -DuplicationStatus(pending=True, files_total=42, files_copied=10) ``` ## Search for Spaces diff --git a/docs/source/en/package_reference/cli.md b/docs/source/en/package_reference/cli.md index ce93fcf7140..183e3acfa6f 100644 --- a/docs/source/en/package_reference/cli.md +++ b/docs/source/en/package_reference/cli.md @@ -3494,7 +3494,6 @@ $ hf repos [OPTIONS] COMMAND [ARGS]... * `delete`: Delete a repo from the Hub. * `delete-files`: Delete files from a repo on the Hub. * `duplicate`: Duplicate a repo on the Hub (model, dataset, or Space). -* `duplication-status`: Get the progress of the background file copy of a duplicated repo. * `list`: List all repos (models, datasets, spaces, buckets) with storage info. [alias: ls] * `move`: Move a repository from a namespace to another namespace. * `settings`: Update the settings of a repository. @@ -3777,34 +3776,6 @@ Learn more Read the documentation at https://huggingface.co/docs/huggingface_hub/en/guides/cli -### `hf repos duplication-status` - -Get the progress of the background file copy of a duplicated repo. - -**Usage**: - -```console -$ hf repos duplication-status [OPTIONS] REPO_ID -``` - -**Arguments**: - -* `REPO_ID`: The ID of the repo (e.g. `username/repo-name` or `spaces/username/repo-name`). [required] - -**Options**: - -* `--type, --repo-type [model|dataset|space|kernel]`: The type of repository (model, dataset, space, or kernel). [default: model] -* `--token TEXT`: A User Access Token generated from https://huggingface.co/settings/tokens. -* `--help`: Show this message and exit. - -Examples - $ hf repos duplication-status myorg/my-gdpval --type dataset - -Learn more - Use `hf --help` for more information about a command. - Read the documentation at https://huggingface.co/docs/huggingface_hub/en/guides/cli - - ### `hf repos list` List all repos (models, datasets, spaces, buckets) with storage info. [alias: ls] diff --git a/docs/source/en/package_reference/hf_api.md b/docs/source/en/package_reference/hf_api.md index e67f61ee795..e3172f67265 100644 --- a/docs/source/en/package_reference/hf_api.md +++ b/docs/source/en/package_reference/hf_api.md @@ -85,10 +85,6 @@ models = hf_api.list_models() [[autodoc]] huggingface_hub.hf_api.DuplicatedRepoUrl -### DuplicationStatus - -[[autodoc]] huggingface_hub.hf_api.DuplicationStatus - ### GitRefInfo [[autodoc]] huggingface_hub.hf_api.GitRefInfo diff --git a/src/huggingface_hub/__init__.py b/src/huggingface_hub/__init__.py index 0b62bdc7458..eae89f63b7b 100644 --- a/src/huggingface_hub/__init__.py +++ b/src/huggingface_hub/__init__.py @@ -197,7 +197,6 @@ "DatasetInfo", "DatasetLeaderboardEntry", "DuplicatedRepoUrl", - "DuplicationStatus", "GitCommitInfo", "GitRefInfo", "GitRefs", @@ -273,7 +272,6 @@ "get_dataset_leaderboard", "get_dataset_tags", "get_discussion_details", - "get_duplication_status", "get_full_repo_name", "get_inference_endpoint", "get_local_safetensors_metadata", @@ -739,7 +737,6 @@ "DocumentQuestionAnsweringParameters", "DryRunFileInfo", "DuplicatedRepoUrl", - "DuplicationStatus", "EvalResult", "EvalResultEntry", "FLAX_WEIGHTS_NAME", @@ -1019,7 +1016,6 @@ "get_dataset_leaderboard", "get_dataset_tags", "get_discussion_details", - "get_duplication_status", "get_full_repo_name", "get_hf_file_metadata", "get_inference_endpoint", @@ -1403,7 +1399,6 @@ def __dir__(): DatasetInfo, # noqa: F401 DatasetLeaderboardEntry, # noqa: F401 DuplicatedRepoUrl, # noqa: F401 - DuplicationStatus, # noqa: F401 GitCommitInfo, # noqa: F401 GitRefInfo, # noqa: F401 GitRefs, # noqa: F401 @@ -1479,7 +1474,6 @@ def __dir__(): get_dataset_leaderboard, # noqa: F401 get_dataset_tags, # noqa: F401 get_discussion_details, # noqa: F401 - get_duplication_status, # noqa: F401 get_full_repo_name, # noqa: F401 get_inference_endpoint, # noqa: F401 get_local_safetensors_metadata, # noqa: F401 diff --git a/src/huggingface_hub/cli/repos.py b/src/huggingface_hub/cli/repos.py index d7bcf28fa76..b433d6fbfb0 100644 --- a/src/huggingface_hub/cli/repos.py +++ b/src/huggingface_hub/cli/repos.py @@ -306,28 +306,10 @@ def repo_duplicate( ) out.result("Repo duplicated", from_id=from_id, to_id=repo_url.repo_id, url=str(repo_url)) if repo_url.files_copy_pending: - type_flag = "" if repo_type == RepoType.model else f" --type {repo_type.value}" out.warning( f"LFS/Xet files are still being copied in the background. Downloading them from '{repo_url.repo_id}'" " may fail until the copy is complete." ) - out.hint(f"Use 'hf repos duplication-status {repo_url.repo_id}{type_flag}' to check the copy progress.") - - -@repos_cli.command( - "duplication-status", - examples=[ - "hf repos duplication-status myorg/my-gdpval --type dataset", - ], -) -def repo_duplication_status( - repo_id: RepoIdArg, - repo_type: RepoTypeOpt = RepoType.model, - token: TokenOpt = None, -) -> None: - """Get the progress of the background file copy of a duplicated repo.""" - api = get_hf_api(token=token) - out.dict(api.get_duplication_status(repo_id, repo_type=repo_type.value, token=token)) @repos_cli.command("delete", examples=["hf repos delete my-model"]) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 09d5d10d1dc..93eaea0677e 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -569,46 +569,15 @@ def __repr__(self) -> str: class DuplicatedRepoUrl(RepoUrl): - """[`RepoUrl`] returned by [`duplicate_repo`], with information about the background file copy. + """[`RepoUrl`] returned by [`duplicate_repo`]. Attributes: files_copy_pending (`bool`): `True` if the LFS/Xet files of the new repo are still being copied in the background. Until the copy is - complete, downloading these files may fail. Use [`get_duplication_status`] to check progress. + complete, downloading these files may fail. """ - def __new__(cls, url: Any, endpoint: str | None = None, files_copy_pending: bool = False): - return super().__new__(cls, url, endpoint=endpoint) - - def __init__(self, url: Any, endpoint: str | None = None, files_copy_pending: bool = False) -> None: - super().__init__(url, endpoint=endpoint) - self.files_copy_pending = files_copy_pending - - -@dataclass -class DuplicationStatus: - """Progress of the background LFS/Xet file copy of a duplicated repo. - - Returned by [`get_duplication_status`]. - - Attributes: - pending (`bool`): - `True` while the LFS/Xet files are still being copied. `False` once the copy is complete (or if the repo - has no copy in progress). - files_total (`int`, *optional*): - Number of LFS/Xet files to copy. Only set while the copy is pending. - files_copied (`int`, *optional*): - Number of files copied so far. Only set while the copy is pending. - """ - - pending: bool - files_total: int | None - files_copied: int | None - - def __init__(self, **kwargs: Any) -> None: - self.pending = kwargs["pending"] - self.files_total = kwargs.get("filesTotal") - self.files_copied = kwargs.get("filesCopied") + files_copy_pending: bool = False def _resolve_copy_target_path( @@ -8663,7 +8632,7 @@ def duplicate_repo( The git history is copied right away, but the LFS/Xet files may still be copied in the background once this method returns. In that case, `files_copy_pending` is `True` on the returned value and downloading these files - from the new repo may fail until the copy is complete. Use [`get_duplication_status`] to check progress. + from the new repo may fail until the copy is complete. Args: from_id (`str`): @@ -8821,59 +8790,9 @@ def duplicate_repo( raise data = r.json() - return DuplicatedRepoUrl( - data["url"], endpoint=self.endpoint, files_copy_pending=data.get("filesCopyPending", False) - ) - - @validate_hf_hub_args - def get_duplication_status( - self, - repo_id: str, - *, - repo_type: str | None = None, - token: bool | str | None = None, - ) -> DuplicationStatus: - """Get the progress of the background LFS/Xet file copy of a repo created with [`duplicate_repo`]. - - When duplicating a repo, the LFS/Xet files are copied in the background. Until the copy is complete, - downloading these files from the new repo may fail. - - Args: - repo_id (`str`): - ID of the new repo (the duplicate, not the original one). Example: `"myorg/my-gdpval"`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if the repo is a dataset or a Space, `None` or `"model"` if it is a - model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`DuplicationStatus`]: The copy status. `pending` is `False` once the copy is complete (or if the repo - has no copy in progress). `files_total` and `files_copied` are only set while the copy is pending. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If the repo cannot be found. This may be because it doesn't exist, or because it is set to `private` - and you do not have access. - - Example: - ```python - >>> from huggingface_hub import duplicate_repo, get_duplication_status - >>> repo_url = duplicate_repo("openai/gdpval", repo_type="dataset") - >>> get_duplication_status(repo_url.repo_id, repo_type="dataset") - DuplicationStatus(pending=True, files_total=42, files_copied=10) - ``` - """ - repo_type = repo_type or constants.REPO_TYPE_MODEL - r = get_session().get( - f"{self.endpoint}/api/{repo_type}s/{repo_id}/duplicate/status", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(r) - return DuplicationStatus(**r.json()) + repo_url = DuplicatedRepoUrl(data["url"], endpoint=self.endpoint) + repo_url.files_copy_pending = data.get("filesCopyPending", False) + return repo_url @validate_hf_hub_args def set_space_volumes( @@ -15301,7 +15220,6 @@ def get_local_safetensors_metadata(path: str | Path) -> SafetensorsRepoMetadata: pause_space = api.pause_space restart_space = api.restart_space duplicate_repo = api.duplicate_repo -get_duplication_status = api.get_duplication_status set_space_volumes = api.set_space_volumes delete_space_volumes = api.delete_space_volumes enable_space_dev_mode = api.enable_space_dev_mode diff --git a/tests/test_cli.py b/tests/test_cli.py index 920b078e0b7..524cc32e912 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -32,7 +32,7 @@ from huggingface_hub.cli.skills import build_skill_md from huggingface_hub.cli.upload import _resolve_upload_paths, upload from huggingface_hub.errors import CLIError, DeviceCodeError, HfUriError, RevisionNotFoundError -from huggingface_hub.hf_api import DuplicationStatus, ModelInfo +from huggingface_hub.hf_api import ModelInfo from huggingface_hub.utils import ( CachedFileInfo, CachedRepoInfo, @@ -1810,25 +1810,6 @@ def test_repo_duplicate_secret_from_env(self, runner: CliRunner) -> None: space_volumes=None, ) - def test_repo_duplicate_files_copy_pending(self, runner: CliRunner) -> None: - with patch("huggingface_hub.cli.repos.get_hf_api") as api_cls: - api_cls.return_value.duplicate_repo.return_value = Mock(repo_id="myorg/my-copy", files_copy_pending=True) - result = runner.invoke(app, ["repos", "duplicate", "owner/repo", "myorg/my-copy", "--type", "dataset"]) - assert result.exit_code == 0 - assert "still being copied in the background" in result.stderr - assert "hf repos duplication-status myorg/my-copy --type dataset" in result.stderr - - def test_repo_duplication_status(self, runner: CliRunner) -> None: - with patch("huggingface_hub.cli.repos.get_hf_api") as api_cls: - api = api_cls.return_value - api.get_duplication_status.return_value = DuplicationStatus(pending=True, filesTotal=42, filesCopied=10) - result = runner.invoke( - app, ["repos", "duplication-status", "myorg/my-copy", "--type", "dataset", "--format", "json"] - ) - assert result.exit_code == 0 - api.get_duplication_status.assert_called_once_with("myorg/my-copy", repo_type="dataset", token=None) - assert json.loads(result.stdout) == {"pending": True, "files_total": 42, "files_copied": 10} - class TestRepoMoveCommand: def test_repo_move_basic(self, runner: CliRunner) -> None: diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index f7ce6b3f1ec..08226f1fbe9 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -57,7 +57,6 @@ CommitInfo, DatasetInfo, DatasetLeaderboardEntry, - DuplicationStatus, ExpandDatasetProperty_T, ExpandModelProperty_T, ExpandSpaceProperty_T, @@ -4634,26 +4633,6 @@ def test_create_inference_endpoint_private_link_payload(mocker): assert "accountId" not in payload -def test_duplicate_repo_files_copy_pending(mocker): - mock_session = mocker.patch("huggingface_hub.hf_api.get_session").return_value - mock_session.post.return_value.json.return_value = { - "url": f"{ENDPOINT_STAGING}/datasets/myorg/my-copy", - "filesCopyPending": True, - } - mock_session.get.return_value.json.return_value = {"pending": True, "filesTotal": 42, "filesCopied": 10} - api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN) - - repo_url = api.duplicate_repo("owner/repo", "myorg/my-copy", repo_type="dataset") - assert isinstance(repo_url, RepoUrl) - assert repo_url.repo_id == "myorg/my-copy" - assert repo_url.files_copy_pending is True - - status = api.get_duplication_status("myorg/my-copy", repo_type="dataset") - assert status == DuplicationStatus(pending=True, filesTotal=42, filesCopied=10) - assert (status.pending, status.files_total, status.files_copied) == (True, 42, 10) - assert mock_session.get.call_args.args[0] == f"{ENDPOINT_STAGING}/api/datasets/myorg/my-copy/duplicate/status" - - @pytest.mark.parametrize( "custom_image, registry_credentials, match", [