diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 31c45f9a1..5caf0056a 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -1,8 +1,9 @@ from __future__ import annotations import optparse +import pathlib from dataclasses import dataclass -from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast +from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Set, cast from pip._internal.cache import WheelCache from pip._internal.index.package_finder import PackageFinder @@ -83,7 +84,23 @@ def parse_requirements( options: optparse.Values | None = None, constraint: bool = False, isolated: bool = False, + comes_from_stdin: bool = False, ) -> Iterator[InstallRequirement]: + # the `comes_from` data will be rewritten in different ways in different conditions + # each rewrite rule is expressible as a str->str function + rewrite_comes_from: Callable[[str], str] + + if comes_from_stdin: + # if data is coming from stdin, then `comes_from="-r -"` + rewrite_comes_from = _rewrite_comes_from_to_hardcoded_stdin_value + elif pathlib.Path(filename).is_absolute(): + # if the input path is absolute, just normalize paths to posix-style + rewrite_comes_from = _normalize_comes_from_location + else: + # if the input was a relative path, set the rewrite rule to rewrite + # absolute paths to be relative + rewrite_comes_from = _relativize_comes_from_location + for parsed_req in _parse_requirements( filename, session, finder=finder, options=options, constraint=constraint ): @@ -94,7 +111,67 @@ def parse_requirements( file_link = FileLink(install_req.link.url) file_link._url = parsed_req.requirement install_req.link = file_link - yield copy_install_requirement(install_req) + install_req = copy_install_requirement(install_req) + + install_req.comes_from = rewrite_comes_from(install_req.comes_from) + + yield install_req + + +def _rewrite_comes_from_to_hardcoded_stdin_value(_: str, /) -> str: + """Produce the hardcoded ``comes_from`` value for stdin.""" + return "-r -" + + +def _relativize_comes_from_location(original_comes_from: str, /) -> str: + """ + Convert a ``comes_from`` path to a relative posix path. + + This is the rewrite rule used when ``-r`` or ``-c`` appears in + ``comes_from`` data with an absolute path. + + The ``-r`` or ``-c`` qualifier is retained, the path is relativized + with respect to the CWD, and the path is converted to posix style. + """ + # require `-r` or `-c` as the source + if not original_comes_from.startswith(("-r ", "-c ")): + return original_comes_from + + # split on the space + prefix, space_sep, suffix = original_comes_from.partition(" ") + + file_path = pathlib.Path(suffix) + + # if the path was not absolute, normalize to posix-style and finish processing + if not file_path.is_absolute(): + return f"{prefix} {file_path.as_posix()}" + + # make it relative to the current working dir + suffix = file_path.relative_to(pathlib.Path.cwd()).as_posix() + return f"{prefix}{space_sep}{suffix}" + + +def _normalize_comes_from_location(original_comes_from: str, /) -> str: + """ + Convert a ``comes_from`` path to a posix-style path. + + This is the rewrite rule when ``-r`` or ``-c`` appears in ``comes_from`` + data and the input path was absolute, meaning we should not relativize the + locations. + + The ``-r`` or ``-c`` qualifier is retained, and the path is converted to + posix style. + """ + # require `-r` or `-c` as the source + if not original_comes_from.startswith(("-r ", "-c ")): + return original_comes_from + + # split on the space + prefix, space_sep, suffix = original_comes_from.partition(" ") + + # convert to a posix-style path + suffix = pathlib.Path(suffix).as_posix() + return f"{prefix}{space_sep}{suffix}" def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache: diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index 5a4448e3f..f585dfc34 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -447,9 +447,13 @@ def _wheel_support_index_min(self: Wheel, tags: list[Tag]) -> int: Wheel.support_index_min = _wheel_support_index_min self._available_candidates_cache = {} - # If we don't clear this cache then it can contain results from an - # earlier call when allow_all_wheels wasn't active. See GH-1532 - self.finder.find_all_candidates.cache_clear() + # Finder internally caches results, and there is no public method to + # clear the cache, so we re-create the object here. If we don't clear + # this cache then it can contain results from an earlier call when + # allow_all_wheels wasn't active. See GH-1532 + self._finder = self.command._build_package_finder( + options=self.options, session=self.session + ) try: yield diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 7c53435c2..a597fe602 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -354,7 +354,6 @@ def cli( # reading requirements from install_requires in setup.py. tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) tmpfile.write(sys.stdin.read()) - comes_from = "-r -" tmpfile.flush() reqs = list( parse_requirements( @@ -362,10 +361,9 @@ def cli( finder=repository.finder, session=repository.session, options=repository.options, + comes_from_stdin=True, ) ) - for req in reqs: - req.comes_from = comes_from constraints.extend(reqs) elif is_setup_file: setup_file_found = True diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index afc9c8776..7f862b5b3 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import hashlib import os import pathlib @@ -7,6 +8,7 @@ import shutil import subprocess import sys +import typing from textwrap import dedent from unittest import mock from unittest.mock import MagicMock @@ -39,6 +41,55 @@ ) +@pytest.fixture(scope="session") +def installed_pip_version(): + return get_pip_version_for_python_executable(sys.executable) + + +@pytest.fixture(scope="session") +def pip_produces_absolute_paths(installed_pip_version): + # in pip v24.3, new normalization will occur because `comes_from` started + # to be normalized to abspaths + return installed_pip_version >= Version("24.3") + + +@dataclasses.dataclass +class TestFilesCollection: + """ + A small data-builder for setting up files in a tmp dir. + + Contains a name for use as the ID in parametrized tests and contents. + 'contents' maps from subpaths in the tmp dir to file content or callables + which produce file content given the tmp dir. + """ + + # the name for the collection of files + name: str + # static or computed contents + contents: dict[str, str | typing.Callable[[pathlib.Path], str]] + + def __str__(self) -> str: + return self.name + + def populate(self, tmp_path: pathlib.Path) -> None: + """Populate the tmp dir with file contents.""" + for path_str, content in self.contents.items(): + path = tmp_path / path_str + path.parent.mkdir(exist_ok=True, parents=True) + if isinstance(content, str): + path.write_text(content) + else: + path.write_text(content(tmp_path)) + + def get_path_to(self, filename: str) -> str: + """Given a filename, find the (first) path to that filename in the contents.""" + return next( + stub_file_path + for stub_file_path in self.contents + if (stub_file_path == filename) or stub_file_path.endswith(f"/{filename}") + ) + + @pytest.fixture( autouse=True, params=[ @@ -1916,7 +1967,7 @@ def test_many_inputs_includes_all_annotations(pip_conf, runner, tmp_path, num_in "small-fake-a==0.1", " # via", ] - + [f" # -r {req_in}" for req_in in req_ins] + + [f" # -r {req_in.as_posix()}" for req_in in req_ins] ) + "\n" ) @@ -3838,3 +3889,156 @@ def test_stdout_should_not_be_read_when_stdin_is_not_a_plain_file( out = runner.invoke(cli, [req_in.as_posix(), "--output-file", fifo.as_posix()]) assert out.exit_code == 0, out + + +@pytest.mark.parametrize( + "input_path_absolute", (True, False), ids=("absolute-input", "relative-input") +) +@pytest.mark.parametrize( + "test_files_collection", + ( + TestFilesCollection( + "relative_include", + { + "requirements2.in": "small-fake-a\n", + "requirements.in": "-r requirements2.in\n", + }, + ), + TestFilesCollection( + "absolute_include", + { + "requirements2.in": "small-fake-a\n", + "requirements.in": lambda tmpdir: f"-r {(tmpdir / 'requirements2.in').as_posix()}", + }, + ), + ), + ids=str, +) +def test_second_order_requirements_path_handling( + pip_conf, + runner, + tmp_path, + monkeypatch, + pip_produces_absolute_paths, + input_path_absolute, + test_files_collection, +): + """ + Test normalization of ``-r`` includes in output. + + Given nested requirements files, the internal requirements file path will + be written in the output, and it will be absolute or relative depending + only on whether or not the initial path was absolute or relative. + """ + test_files_collection.populate(tmp_path) + + # the input path is given on the CLI as absolute or relative + # and this determines the expected output path as well + input_dir_path = tmp_path if input_path_absolute else pathlib.Path(".") + input_path = (input_dir_path / "requirements.in").as_posix() + output_path = (input_dir_path / "requirements2.in").as_posix() + + with monkeypatch.context() as revertable_ctx: + revertable_ctx.chdir(tmp_path) + + out = runner.invoke( + cli, + [ + "--output-file", + "-", + "--quiet", + "--no-header", + "--no-emit-options", + "-r", + input_path, + ], + ) + + assert out.exit_code == 0 + assert out.stdout == dedent( + f"""\ + small-fake-a==0.2 + # via -r {output_path} + """ + ) + + +@pytest.mark.parametrize( + "test_files_collection", + ( + TestFilesCollection( + "parent_dir", + { + "requirements2.in": "small-fake-a\n", + "subdir/requirements.in": "-r ../requirements2.in\n", + }, + ), + TestFilesCollection( + "subdir", + { + "requirements.in": "-r ./subdir/requirements2.in", + "subdir/requirements2.in": "small-fake-a\n", + }, + ), + TestFilesCollection( + "sibling_dir", + { + "subdir1/requirements.in": "-r ../subdir2/requirements2.in", + "subdir2/requirements2.in": "small-fake-a\n", + }, + ), + ), + ids=str, +) +def test_second_order_requirements_relative_path_in_separate_dir( + pip_conf, + runner, + tmp_path, + monkeypatch, + test_files_collection, + pip_produces_absolute_paths, +): + """ + Test normalization of ``-r`` includes when the requirements files are in + distinct directories. + + Confirm that the output path will be relative to the current working + directory. + """ + test_files_collection.populate(tmp_path) + # the input is the path to 'requirements.in' relative to the starting dir + input_path = test_files_collection.get_path_to("requirements.in") + # the output should also be relative to the starting dir, the path to 'requirements2.in' + output_path = test_files_collection.get_path_to("requirements2.in") + + # for older pip versions, recompute the output path to be relative to the input path + if not pip_produces_absolute_paths: + # traverse upwards to the root tmp dir, and append the output path to that + # similar to pathlib.Path.relative_to(..., walk_up=True) + relative_segments = len(pathlib.Path(input_path).parents) - 1 + output_path = ( + pathlib.Path(input_path).parent / ("../" * relative_segments) / output_path + ).as_posix() + + with monkeypatch.context() as revertable_ctx: + revertable_ctx.chdir(tmp_path) + out = runner.invoke( + cli, + [ + "--output-file", + "-", + "--quiet", + "--no-header", + "--no-emit-options", + "-r", + input_path, + ], + ) + + assert out.exit_code == 0 + assert out.stdout == dedent( + f"""\ + small-fake-a==0.2 + # via -r {output_path} + """ + ) diff --git a/tox.ini b/tox.ini index dba0b4bb2..fbb9f2ec5 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,9 @@ extras = testing coverage: coverage deps = - pipsupported: pip==24.2 - pipsupported: setuptools <= 75.8.2 + pipsupported: pip == 24.3.1 ; python_version < "3.9" + pipsupported: pip == 25.1.1 ; python_version >= "3.9" + pipsupported: setuptools <= 80.0 piplowest: pip == 22.2.* ; python_version < "3.12" piplowest: pip == 23.2.* ; python_version >= "3.12"