diff --git a/news/9081.feature.rst b/news/9081.feature.rst new file mode 100644 index 00000000000..b4d7a1d55ce --- /dev/null +++ b/news/9081.feature.rst @@ -0,0 +1,6 @@ +Add ``--use-feature inprocess-build-deps`` to request that build dependencies are installed +within the same pip install process. This new mechanism is faster, supports ``--no-clean`` +and ``--no-cache-dir`` reliably, and supports prompting for authentication. + +Enabling this feature will also enable ``--use-feature build-constraints``. This feature will +become the default in a future pip version. diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index f28d862f279..b4cf423461d 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -9,25 +9,37 @@ import sys import textwrap from collections import OrderedDict -from collections.abc import Iterable +from collections.abc import Iterable, Sequence +from contextlib import AbstractContextManager as ContextManager +from contextlib import nullcontext +from io import StringIO from types import TracebackType from typing import TYPE_CHECKING, Protocol, TypedDict from pip._vendor.packaging.version import Version from pip import __file__ as pip_location -from pip._internal.cli.spinners import open_spinner +from pip._internal.cli.spinners import open_rich_spinner, open_spinner +from pip._internal.exceptions import ( + BuildDependencyInstallError, + DiagnosticPipError, + InstallWheelBuildError, + PipError, +) from pip._internal.locations import get_platlib, get_purelib, get_scheme from pip._internal.metadata import get_default_environment, get_environment from pip._internal.utils.deprecation import deprecated -from pip._internal.utils.logging import VERBOSE +from pip._internal.utils.logging import VERBOSE, capture_logging from pip._internal.utils.packaging import get_requirement from pip._internal.utils.subprocess import call_subprocess from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds if TYPE_CHECKING: + from pip._internal.cache import WheelCache from pip._internal.index.package_finder import PackageFinder + from pip._internal.operations.build.build_tracker import BuildTracker from pip._internal.req.req_install import InstallRequirement + from pip._internal.resolution.base import BaseResolver class ExtraEnviron(TypedDict, total=False): extra_environ: dict[str, str] @@ -245,6 +257,177 @@ def install( ) +class InprocessBuildEnvironmentInstaller: + """ + Build dependency installer that runs in the same pip process. + + This contains a stripped down version of the install command with + only the logic necessary for installing build dependencies. The + finder, session, build tracker, and wheel cache are reused, but new + instances of everything else are created as needed. + + Options are inherited from the parent install command unless + they don't make sense for build dependencies (in which case, they + are hard-coded, see comments below). + """ + + def __init__( + self, + *, + finder: PackageFinder, + build_tracker: BuildTracker, + wheel_cache: WheelCache, + build_constraints: Sequence[InstallRequirement] = (), + verbosity: int = 0, + ) -> None: + from pip._internal.operations.prepare import RequirementPreparer + + self._finder = finder + self._build_constraints = build_constraints + self._wheel_cache = wheel_cache + self._level = 0 + + build_dir = TempDirectory(kind="build-env-install", globally_managed=True) + self._preparer = RequirementPreparer( + build_isolation_installer=self, + # Inherited options or state. + finder=finder, + session=finder._link_collector.session, + build_dir=build_dir.path, + build_tracker=build_tracker, + verbosity=verbosity, + # This is irrelevant as it only applies to editable requirements. + src_dir="", + # Hard-coded options (that should NOT be inherited). + download_dir=None, + build_isolation=True, + check_build_deps=False, + progress_bar="off", + # TODO: hash-checking should be extended to build deps, but that is + # deferred for later as it'd be a breaking change. + require_hashes=False, + use_user_site=False, + lazy_wheel=False, + legacy_resolver=False, + ) + + def install( + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement | None, + ) -> None: + """Install entrypoint. Manages output capturing and error handling.""" + capture_logs = not logger.isEnabledFor(VERBOSE) and self._level == 0 + if capture_logs: + # Hide the logs from the installation of build dependencies. + # They will be shown only if an error occurs. + capture_ctx: ContextManager[StringIO] = capture_logging() + spinner: ContextManager[None] = open_rich_spinner(f"Installing {kind}") + else: + # Otherwise, pass-through all logs (with a header). + capture_ctx, spinner = nullcontext(StringIO()), nullcontext() + logger.info("Installing %s ...", kind) + + try: + self._level += 1 + with spinner, capture_ctx as stream: + self._install_impl(requirements, prefix) + + except DiagnosticPipError as exc: + # Format similar to a nested subprocess error, where the + # causing error is shown first, followed by the build error. + logger.info(textwrap.dedent(stream.getvalue())) + logger.error("%s", exc, extra={"rich": True}) + logger.info("") + raise BuildDependencyInstallError( + for_req, requirements, cause=exc, log_lines=None + ) + + except Exception as exc: + logs: list[str] | None = textwrap.dedent(stream.getvalue()).splitlines() + if not capture_logs: + # If logs aren't being captured, then display the error inline + # with the rest of the logs. + logs = None + if isinstance(exc, PipError): + logger.error("%s", exc) + else: + logger.exception("pip crashed unexpectedly") + raise BuildDependencyInstallError( + for_req, requirements, cause=exc, log_lines=logs + ) + + finally: + self._level -= 1 + + def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: + """Core build dependency install logic.""" + from pip._internal.commands.install import installed_packages_summary + from pip._internal.req import install_given_reqs + from pip._internal.req.constructors import install_req_from_line + from pip._internal.wheel_builder import build + + ireqs = [install_req_from_line(req, user_supplied=True) for req in requirements] + ireqs.extend(self._build_constraints) + + resolver = self._make_resolver() + resolved_set = resolver.resolve(ireqs, check_supported_wheels=True) + self._preparer.prepare_linked_requirements_more( + resolved_set.requirements.values() + ) + + reqs_to_build = [ + r for r in resolved_set.requirements_to_install if not r.is_wheel + ] + _, build_failures = build(reqs_to_build, self._wheel_cache, verify=True) + if build_failures: + raise InstallWheelBuildError(build_failures) + + installed = install_given_reqs( + resolver.get_installation_order(resolved_set), + prefix=prefix.path, + # Hard-coded options (that should NOT be inherited). + root=None, + home=None, + warn_script_location=False, + use_user_site=False, + # As the build environment is ephemeral, it's wasteful to + # pre-compile everything since not all modules will be used. + pycompile=False, + progress_bar="off", + ) + + env = get_environment(list(prefix.lib_dirs)) + if summary := installed_packages_summary(installed, env): + logger.info(summary) + + def _make_resolver(self) -> BaseResolver: + """Create a new resolver for one time use.""" + # Legacy installer never used the legacy resolver so create a + # resolvelib resolver directly. Yuck. + from pip._internal.req.constructors import install_req_from_req_string + from pip._internal.resolution.resolvelib.resolver import Resolver + + return Resolver( + make_install_req=install_req_from_req_string, + # Inherited state. + preparer=self._preparer, + finder=self._finder, + wheel_cache=self._wheel_cache, + # Hard-coded options (that should NOT be inherited). + ignore_requires_python=False, + use_user_site=False, + ignore_dependencies=False, + ignore_installed=True, + force_reinstall=False, + upgrade_strategy="to-satisfy-only", + py_version_info=None, + ) + + class BuildEnvironment: """Creates and manages an isolated environment to install build deps""" diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 7acc29cb349..499a46f9640 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -235,6 +235,17 @@ def _main(self, args: list[str]) -> int: ) options.cache_dir = None + if ( + "inprocess-build-deps" in options.features_enabled + and os.environ.get("PIP_CONSTRAINT", "") + and "build-constraint" not in options.features_enabled + ): + logger.warning( + "In-process build dependencies are enabled, " + "PIP_CONSTRAINT will have no effect for build dependencies" + ) + options.features_enabled.append("build-constraint") + return self._run_wrapper(level_number, options, args) def handler_map(self) -> dict[str, Callable[[Values, list[str]], None]]: diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 6ac73a0b6eb..fc2c75d06b5 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -1056,6 +1056,7 @@ def check_list_path_option(options: Values) -> None: choices=[ "fast-deps", "build-constraint", + "inprocess-build-deps", ] + ALWAYS_ENABLED_FEATURES, help="Enable new functionality, that may be backward incompatible.", diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index ca7be2f714a..84437a2a57d 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -13,7 +13,11 @@ from optparse import Values from typing import Any, Callable, TypeVar -from pip._internal.build_env import SubprocessBuildEnvironmentInstaller +from pip._internal.build_env import ( + BuildEnvironmentInstaller, + InprocessBuildEnvironmentInstaller, + SubprocessBuildEnvironmentInstaller, +) from pip._internal.cache import WheelCache from pip._internal.cli import cmdoptions from pip._internal.cli.cmdoptions import make_target_python @@ -100,6 +104,31 @@ def wrapper(self: _CommandT, options: Values, args: list[str]) -> int: return wrapper +def parse_constraint_files( + constraint_files: list[str], + finder: PackageFinder, + options: Values, + session: PipSession, +) -> list[InstallRequirement]: + requirements = [] + for filename in constraint_files: + for parsed_req in parse_requirements( + filename, + constraint=True, + finder=finder, + options=options, + session=session, + ): + req_to_add = install_req_from_parsed_requirement( + parsed_req, + isolated=options.isolated_mode, + user_supplied=False, + ) + requirements.append(req_to_add) + + return requirements + + class RequirementCommand(IndexGroupCommand): def __init__(self, *args: Any, **kw: Any) -> None: super().__init__(*args, **kw) @@ -159,16 +188,31 @@ def make_requirement_preparer( "build-constraint" in options.features_enabled ) + env_installer: BuildEnvironmentInstaller + if "inprocess-build-deps" in options.features_enabled: + build_constraint_reqs = parse_constraint_files( + build_constraints, finder, options, session + ) + env_installer = InprocessBuildEnvironmentInstaller( + finder=finder, + build_tracker=build_tracker, + build_constraints=build_constraint_reqs, + verbosity=verbosity, + wheel_cache=WheelCache(options.cache_dir), + ) + else: + env_installer = SubprocessBuildEnvironmentInstaller( + finder, + build_constraints=build_constraints, + build_constraint_feature_enabled=build_constraint_feature_enabled, + ) + return RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, download_dir=download_dir, build_isolation=options.build_isolation, - build_isolation_installer=SubprocessBuildEnvironmentInstaller( - finder, - build_constraints=build_constraints, - build_constraint_feature_enabled=build_constraint_feature_enabled, - ), + build_isolation_installer=env_installer, check_build_deps=options.check_build_deps, build_tracker=build_tracker, session=session, @@ -251,22 +295,14 @@ def get_requirements( requirements: list[InstallRequirement] = [] if not should_ignore_regular_constraints(options): - for filename in options.constraints: - for parsed_req in parse_requirements( - filename, - constraint=True, - finder=finder, - options=options, - session=session, - ): - req_to_add = install_req_from_parsed_requirement( - parsed_req, - isolated=options.isolated_mode, - user_supplied=False, - ) - requirements.append(req_to_add) + constraints = parse_constraint_files( + options.constraints, finder, options, session + ) + requirements.extend(constraints) for req in args: + if not req.strip(): + continue req_to_add = install_req_from_line( req, comes_from=None, diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index e815c51f6c1..7bd05466a6f 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -34,11 +34,11 @@ InstallWheelBuildError, ) from pip._internal.locations import get_scheme -from pip._internal.metadata import get_environment +from pip._internal.metadata import BaseEnvironment, get_environment from pip._internal.models.installation_report import InstallationReport from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.operations.check import ConflictDetails, check_install_conflicts -from pip._internal.req import install_given_reqs +from pip._internal.req import InstallationResult, install_given_reqs from pip._internal.req.req_install import ( InstallRequirement, ) @@ -476,34 +476,13 @@ def run(self, options: Values, args: list[str]) -> int: ) env = get_environment(lib_locations) - # Display a summary of installed packages, with extra care to - # display a package name as it was requested by the user. - installed.sort(key=operator.attrgetter("name")) - summary = [] - installed_versions = {} - for distribution in env.iter_all_distributions(): - installed_versions[distribution.canonical_name] = distribution.version - for package in installed: - display_name = package.name - version = installed_versions.get(canonicalize_name(display_name), None) - if version: - text = f"{display_name}-{version}" - else: - text = display_name - summary.append(text) - if conflicts is not None: self._warn_about_conflicts( conflicts, resolver_variant=self.determine_resolver_variant(options), ) - - installed_desc = " ".join(summary) - if installed_desc: - write_output( - "Successfully installed %s", - installed_desc, - ) + if summary := installed_packages_summary(installed, env): + write_output(summary) except OSError as error: show_traceback = self.verbosity >= 1 @@ -642,6 +621,30 @@ def _warn_about_conflicts( logger.critical("\n".join(parts)) +def installed_packages_summary( + installed: list[InstallationResult], env: BaseEnvironment +) -> str: + # Format a summary of installed packages, with extra care to + # display a package name as it was requested by the user. + installed.sort(key=operator.attrgetter("name")) + summary = [] + installed_versions = {} + for distribution in env.iter_all_distributions(): + installed_versions[distribution.canonical_name] = distribution.version + for package in installed: + display_name = package.name + version = installed_versions.get(canonicalize_name(display_name), None) + if version: + text = f"{display_name}-{version}" + else: + text = display_name + summary.append(text) + + if not summary: + return "" + return f"Successfully installed {' '.join(summary)}" + + def get_lib_location_guesses( user: bool = False, home: str | None = None, diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index 8bcb46212f8..4d4e012cc8c 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -14,7 +14,8 @@ import pathlib import re import sys -from collections.abc import Iterator +import traceback +from collections.abc import Iterable, Iterator from itertools import chain, groupby, repeat from typing import TYPE_CHECKING, Literal @@ -896,3 +897,51 @@ def __init__(self, failed: list[InstallRequirement]) -> None: context=", ".join(r.name for r in failed), # type: ignore hint_stmt=None, ) + + +class BuildDependencyInstallError(DiagnosticPipError): + """Raised when build dependencies cannot be installed.""" + + reference = "failed-build-dependency-install" + + def __init__( + self, + req: InstallRequirement | None, + build_reqs: Iterable[str], + *, + cause: Exception, + log_lines: list[str] | None, + ) -> None: + if isinstance(cause, PipError): + note = "This is likely not a problem with pip." + else: + note = ( + "pip crashed unexpectedly. Please file an issue on pip's issue " + "tracker: https://github.com/pypa/pip/issues/new" + ) + + if log_lines is None: + # No logs are available, they must have been printed earlier. + context = Text("See above for more details.") + else: + if isinstance(cause, PipError): + log_lines.append(f"ERROR: {cause}") + else: + # Split rendered error into real lines without trailing newlines. + log_lines.extend( + "".join(traceback.format_exception(cause)).splitlines() + ) + + context = Text.assemble( + f"Installing {' '.join(build_reqs)}\n", + (f"[{len(log_lines)} lines of output]\n", "red"), + "\n".join(log_lines), + ("\n[end of output]", "red"), + ) + + message = Text("Cannot install build dependencies", "green") + if req: + message += Text(f" for {req}") + super().__init__( + message=message, context=context, hint_stmt=None, note_stmt=note + ) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 5cdbeb7f753..e0677035802 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -9,7 +9,7 @@ import threading from collections.abc import Generator from dataclasses import dataclass -from io import TextIOWrapper +from io import StringIO, TextIOWrapper from logging import Filter from typing import Any, ClassVar @@ -29,7 +29,7 @@ from pip._internal.utils._log import VERBOSE, getLogger from pip._internal.utils.compat import WINDOWS from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX -from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.misc import StreamWrapper, ensure_dir _log_state = threading.local() _stdout_console = None @@ -56,6 +56,38 @@ def _is_broken_pipe_error(exc_class: type[BaseException], exc: BaseException) -> return isinstance(exc, OSError) and exc.errno in (errno.EINVAL, errno.EPIPE) +@contextlib.contextmanager +def capture_logging() -> Generator[StringIO, None, None]: + """Capture all pip logs in a buffer temporarily.""" + # Patching sys.std(out|err) directly is not viable as the caller + # may want to emit non-logging output (e.g. a rich spinner). To + # avoid capturing that, temporarily patch the root logging handlers + # to use new rich consoles that write to a StringIO. + handlers = {} + for handler in logging.getLogger().handlers: + if isinstance(handler, RichPipStreamHandler): + # Also store the handler's original console so it can be + # restored on context exit. + handlers[handler] = handler.console + + fake_stream = StreamWrapper.from_stream(sys.stdout) + if not handlers: + yield fake_stream + return + + # HACK: grab no_color attribute from a random handler console since + # it's a global option anyway. + no_color = next(iter(handlers.values())).no_color + fake_console = PipConsole(file=fake_stream, no_color=no_color, soft_wrap=True) + try: + for handler in handlers: + handler.console = fake_console + yield fake_stream + finally: + for handler, original_console in handlers.items(): + handler.console = original_console + + @contextlib.contextmanager def indent_log(num: int = 2) -> Generator[None, None, None]: """ diff --git a/tests/functional/test_build_env.py b/tests/functional/test_build_env.py index ec1b59f068c..9894b5a77b7 100644 --- a/tests/functional/test_build_env.py +++ b/tests/functional/test_build_env.py @@ -2,15 +2,23 @@ import os import sys +from collections.abc import Generator +from contextlib import contextmanager from textwrap import dedent +from typing import Literal import pytest from pip._internal.build_env import ( BuildEnvironment, + BuildEnvironmentInstaller, + InprocessBuildEnvironmentInstaller, SubprocessBuildEnvironmentInstaller, _get_system_sitepackages, ) +from pip._internal.cache import WheelCache +from pip._internal.index.package_finder import PackageFinder +from pip._internal.operations.build.build_tracker import get_build_tracker from tests.lib import ( PipTestEnvironment, @@ -19,15 +27,36 @@ make_test_finder, ) +InstallMethod = Literal["subprocess", "inprocess"] +with_both_installers = pytest.mark.parametrize( + "install_method", ["subprocess", "inprocess"] +) + def indent(text: str, prefix: str) -> str: return "\n".join((prefix if line else "") + line for line in text.split("\n")) +@contextmanager +def make_test_build_env_installer( + method: InstallMethod, finder: PackageFinder +) -> Generator[BuildEnvironmentInstaller]: + if method == "subprocess": + yield SubprocessBuildEnvironmentInstaller(finder) + else: + with get_build_tracker() as tracker: + yield InprocessBuildEnvironmentInstaller( + finder=finder, + build_tracker=tracker, + wheel_cache=WheelCache(None), # type: ignore + ) + + def run_with_build_env( script: PipTestEnvironment, setup_script_contents: str, test_script_contents: str | None = None, + install_method: InstallMethod = "subprocess", ) -> TestPipResult: build_env_script = script.scratch_path / "build_env.py" scratch_path = str(script.scratch_path) @@ -39,14 +68,17 @@ def run_with_build_env( from pip._internal.build_env import ( BuildEnvironment, + InprocessBuildEnvironmentInstaller, SubprocessBuildEnvironmentInstaller, ) + from pip._internal.cache import WheelCache from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import ( SelectionPreferences ) + from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.network.session import PipSession from pip._internal.utils.temp_dir import global_tempdir_manager @@ -62,10 +94,16 @@ def run_with_build_env( selection_prefs=selection_prefs, ) - with global_tempdir_manager(): - build_env = BuildEnvironment( - SubprocessBuildEnvironmentInstaller(finder) - ) + with global_tempdir_manager(), get_build_tracker() as tracker: + if "{install_method}" == "subprocess": + installer = SubprocessBuildEnvironmentInstaller(finder) + else: + installer = InprocessBuildEnvironmentInstaller( + finder=finder, + build_tracker=tracker, + wheel_cache=WheelCache(None), + ) + build_env = BuildEnvironment(installer) """ ) + indent(dedent(setup_script_contents), " ") @@ -88,28 +126,40 @@ def run_with_build_env( return script.run(*args) -def test_build_env_allow_empty_requirements_install() -> None: +@with_both_installers +def test_build_env_allow_empty_requirements_install( + install_method: InstallMethod, +) -> None: finder = make_test_finder() - build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) - for prefix in ("normal", "overlay"): - build_env.install_requirements([], prefix, kind="Installing build dependencies") + with make_test_build_env_installer(install_method, finder) as installer: + build_env = BuildEnvironment(installer) + for prefix in ("normal", "overlay"): + build_env.install_requirements( + [], prefix, kind="Installing build dependencies" + ) -def test_build_env_allow_only_one_install(script: PipTestEnvironment) -> None: +@with_both_installers +def test_build_env_allow_only_one_install( + script: PipTestEnvironment, install_method: InstallMethod +) -> None: create_basic_wheel_for_package(script, "foo", "1.0") create_basic_wheel_for_package(script, "bar", "1.0") finder = make_test_finder(find_links=[os.fspath(script.scratch_path)]) - build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) - for prefix in ("normal", "overlay"): - build_env.install_requirements( - ["foo"], prefix, kind=f"installing foo in {prefix}" - ) - with pytest.raises(AssertionError): + with make_test_build_env_installer(install_method, finder) as installer: + build_env = BuildEnvironment(installer) + for prefix in ("normal", "overlay"): build_env.install_requirements( - ["bar"], prefix, kind=f"installing bar in {prefix}" + ["foo"], prefix, kind=f"installing foo in {prefix}" ) - with pytest.raises(AssertionError): - build_env.install_requirements([], prefix, kind=f"installing in {prefix}") + with pytest.raises(AssertionError): + build_env.install_requirements( + ["bar"], prefix, kind=f"installing bar in {prefix}" + ) + with pytest.raises(AssertionError): + build_env.install_requirements( + [], prefix, kind=f"installing in {prefix}" + ) def test_build_env_requirements_check(script: PipTestEnvironment) -> None: @@ -191,7 +241,10 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: ) -def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> None: +@with_both_installers +def test_build_env_overlay_prefix_has_priority( + script: PipTestEnvironment, install_method: InstallMethod +) -> None: create_basic_wheel_for_package(script, "pkg", "2.0") create_basic_wheel_for_package(script, "pkg", "4.3") result = run_with_build_env( @@ -205,6 +258,7 @@ def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> No """ print(__import__('pkg').__version__) """, + install_method=install_method, ) assert result.stdout.strip() == "2.0", str(result) @@ -234,8 +288,11 @@ def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> No """ +@with_both_installers @pytest.mark.usefixtures("enable_user_site") -def test_build_env_isolation(script: PipTestEnvironment) -> None: +def test_build_env_isolation( + script: PipTestEnvironment, install_method: InstallMethod +) -> None: # Create dummy `pkg` wheel. pkg_whl = create_basic_wheel_for_package(script, "pkg", "1.0") @@ -281,4 +338,5 @@ def test_build_env_isolation(script: PipTestEnvironment) -> None: assert system_path not in normalized_path, \ f"{{system_path}} found in {{normalized_path}}" """, + install_method=install_method, ) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 3b949baa6a6..5fa3b7599ab 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -222,8 +222,9 @@ def test_pep518_with_user_pip( ) +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_pep518_with_extra_and_markers( - script: PipTestEnvironment, data: TestData, common_wheels: Path + script: PipTestEnvironment, data: TestData, common_wheels: Path, flag: str ) -> None: script.pip( "wheel", @@ -233,11 +234,13 @@ def test_pep518_with_extra_and_markers( "-f", data.find_links, data.src.joinpath("pep518_with_extra_and_markers-1.0"), + flag, ) +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_pep518_with_namespace_package( - script: PipTestEnvironment, data: TestData, common_wheels: Path + script: PipTestEnvironment, data: TestData, common_wheels: Path, flag: str ) -> None: script.pip( "wheel", @@ -247,6 +250,7 @@ def test_pep518_with_namespace_package( "-f", data.find_links, data.src.joinpath("pep518_with_namespace_package-1.0"), + flag, use_module=True, ) @@ -256,12 +260,14 @@ def test_pep518_with_namespace_package( "package", ["pep518_forkbomb", "pep518_twin_forkbombs_first", "pep518_twin_forkbombs_second"], ) +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_pep518_forkbombs( script: PipTestEnvironment, data: TestData, common_wheels: Path, command: str, package: str, + flag: str, ) -> None: package_source = next(data.packages.glob(package + "-[0-9]*.tar.gz")) result = script.pip( @@ -273,6 +279,7 @@ def test_pep518_forkbombs( "-f", data.find_links, package, + flag, expect_error=True, ) assert ( @@ -1787,11 +1794,14 @@ def test_install_no_binary_builds_wheels( @pytest.mark.network +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_install_no_binary_builds_pep_517_wheel( - script: PipTestEnvironment, data: TestData + script: PipTestEnvironment, data: TestData, flag: str ) -> None: to_install = data.packages.joinpath("pep517_setup_and_pyproject") - res = script.pip("install", "--no-binary=:all:", "-f", data.find_links, to_install) + res = script.pip( + "install", "--no-binary=:all:", "-f", data.find_links, to_install, flag + ) expected = "Successfully installed pep517-setup-and-pyproject" # Must have installed the package assert expected in str(res), str(res) @@ -1800,12 +1810,15 @@ def test_install_no_binary_builds_pep_517_wheel( @pytest.mark.network +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_install_no_binary_uses_local_backend( - script: PipTestEnvironment, data: TestData, tmpdir: Path + script: PipTestEnvironment, data: TestData, tmpdir: Path, flag: str ) -> None: to_install = data.packages.joinpath("pep517_wrapper_buildsys") script.environ["PIP_TEST_MARKER_FILE"] = marker = str(tmpdir / "marker") - res = script.pip("install", "--no-binary=:all:", "-f", data.find_links, to_install) + res = script.pip( + "install", "--no-binary=:all:", "-f", data.find_links, to_install, flag + ) expected = "Successfully installed pep517-wrapper-buildsys" # Must have installed the package assert expected in str(res), str(res) @@ -1813,8 +1826,9 @@ def test_install_no_binary_uses_local_backend( assert os.path.isfile(marker), "Local PEP 517 backend not used" +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_install_no_binary_uses_cached_wheels( - script: PipTestEnvironment, data: TestData + script: PipTestEnvironment, data: TestData, flag: str ) -> None: # Seed the cache script.pip( @@ -1829,6 +1843,7 @@ def test_install_no_binary_uses_cached_wheels( "-f", data.find_links, "upper", + flag, expect_stderr=True, ) assert "Successfully installed upper-2.0" in str(res), str(res) @@ -2402,11 +2417,13 @@ def test_install_sends_client_cert( assert environ["SSL_CLIENT_CERT"] +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_install_sends_certs_for_pep518_deps( script: PipTestEnvironment, cert_factory: CertFactory, data: TestData, common_wheels: Path, + flag: str, ) -> None: cert_path = cert_factory() ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) @@ -2422,7 +2439,7 @@ def test_install_sends_certs_for_pep518_deps( ] url = f"https://{server.host}:{server.port}/simple" - args = ["install", str(data.packages / "pep517_setup_and_pyproject")] + args = ["install", str(data.packages / "pep517_setup_and_pyproject"), flag] args.extend(["--index-url", url]) args.extend(["--cert", cert_path, "--client-cert", cert_path]) diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index e36c5f80277..c5cc00209c7 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any +import pytest import tomli_w from pip._internal.build_env import ( @@ -103,19 +104,21 @@ def test_backend_path_and_dep(tmpdir: Path, data: TestData) -> None: assert req.pep517_backend.build_wheel("dir") == "Backend called" +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_pep517_install( - script: PipTestEnvironment, tmpdir: Path, data: TestData + script: PipTestEnvironment, tmpdir: Path, data: TestData, flag: str ) -> None: """Check we can build with a custom backend""" project_dir = make_project( tmpdir, requires=["test_backend"], backend="test_backend" ) - result = script.pip("install", "--no-index", "-f", data.backends, project_dir) + result = script.pip("install", "--no-index", "-f", data.backends, project_dir, flag) result.assert_installed("project", editable=False) +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_pep517_install_with_reqs( - script: PipTestEnvironment, tmpdir: Path, data: TestData + script: PipTestEnvironment, tmpdir: Path, data: TestData, flag: str ) -> None: """Backend generated requirements are installed in the build env""" project_dir = make_project( @@ -123,7 +126,14 @@ def test_pep517_install_with_reqs( ) project_dir.joinpath("backend_reqs.txt").write_text("simplewheel") result = script.pip( - "install", "--no-index", "-f", data.backends, "-f", data.packages, project_dir + "install", + "--no-index", + "-f", + data.backends, + "-f", + data.packages, + project_dir, + flag, ) result.assert_installed("project", editable=False) @@ -247,8 +257,9 @@ def test_pep517_backend_requirements_satisfied_by_prerelease( assert "Installing backend dependencies:" not in result.stdout +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_pep517_backend_requirements_already_satisfied( - script: PipTestEnvironment, tmpdir: Path, data: TestData + script: PipTestEnvironment, tmpdir: Path, data: TestData, flag: str ) -> None: project_dir = make_project( tmpdir, requires=["test_backend", "simplewheel==1.0"], backend="test_backend" @@ -262,12 +273,14 @@ def test_pep517_backend_requirements_already_satisfied( "-f", data.packages, project_dir, + flag, ) assert "Installing backend dependencies:" not in result.stdout +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_pep517_install_with_no_cache_dir( - script: PipTestEnvironment, tmpdir: Path, data: TestData + script: PipTestEnvironment, tmpdir: Path, data: TestData, flag: str ) -> None: """Check builds with a custom backends work, even with no cache.""" project_dir = make_project( @@ -280,6 +293,7 @@ def test_pep517_install_with_no_cache_dir( "-f", data.backends, project_dir, + flag, ) result.assert_installed("project", editable=False) @@ -370,3 +384,15 @@ def test_explicit_setuptools_backend( project_dir, ) result.assert_installed(name, editable=False) + + +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) +@pytest.mark.network +def test_nested_builds(script: PipTestEnvironment, flag: str) -> None: + """Smoke test ensuring that nested PEP 517 builds work.""" + # trove-classifiers -> setuptools + # -> calvar -> setuptools + result = script.pip( + "install", "trove-classifiers", "--no-cache", "--no-binary", ":all:", flag + ) + result.assert_installed("trove_classifiers", editable=False) diff --git a/tests/functional/test_proxy.py b/tests/functional/test_proxy.py index 0810d82d3af..434610cef4f 100644 --- a/tests/functional/test_proxy.py +++ b/tests/functional/test_proxy.py @@ -100,8 +100,12 @@ def test_proxy_does_not_override_netrc( strict=False, ) @pytest.mark.network +@pytest.mark.parametrize("flag", ["", "--use-feature=inprocess-build-deps"]) def test_build_deps_use_proxy_from_cli( - script: PipTestEnvironment, capfd: pytest.CaptureFixture[str], data: TestData + script: PipTestEnvironment, + capfd: pytest.CaptureFixture[str], + data: TestData, + flag: str, ) -> None: with proxy.Proxy(port=0, num_acceptors=1, plugins=[AccessLogPlugin]) as proxy1: result = script.pip( @@ -110,6 +114,7 @@ def test_build_deps_use_proxy_from_cli( str(data.packages / "pep517_setup_and_pyproject"), "--proxy", f"http://127.0.0.1:{proxy1.flags.port}", + flag, ) wheel_path = script.scratch / "pep517_setup_and_pyproject-1.0-py3-none-any.whl"