Install build dependencies in-process

news/9081.feature.rst

@@ -0,0 +1,6 @@
+Add ``--use-feature inprocess-build-deps`` to request that build dependencies are installed
+within the same pip install process. This new mechanism is faster, supports ``--no-clean``
+and ``--no-cache-dir`` reliably, and supports prompting for authentication.
+
+Enabling this feature will also enable ``--use-feature build-constraints``. This feature will
+become the default in a future pip version.

src/pip/_internal/build_env.py

@@ -9,25 +9,37 @@
 import sys
 import textwrap
 from collections import OrderedDict
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
+from contextlib import AbstractContextManager as ContextManager
+from contextlib import nullcontext
+from io import StringIO
 from types import TracebackType
 from typing import TYPE_CHECKING, Protocol, TypedDict
 
 from pip._vendor.packaging.version import Version
 
 from pip import __file__ as pip_location
-from pip._internal.cli.spinners import open_spinner
+from pip._internal.cli.spinners import open_rich_spinner, open_spinner
+from pip._internal.exceptions import (
+    BuildDependencyInstallError,
+    DiagnosticPipError,
+    InstallWheelBuildError,
+    PipError,
+)
 from pip._internal.locations import get_platlib, get_purelib, get_scheme
 from pip._internal.metadata import get_default_environment, get_environment
 from pip._internal.utils.deprecation import deprecated
-from pip._internal.utils.logging import VERBOSE
+from pip._internal.utils.logging import VERBOSE, capture_logging
 from pip._internal.utils.packaging import get_requirement
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
 
 if TYPE_CHECKING:
+    from pip._internal.cache import WheelCache
     from pip._internal.index.package_finder import PackageFinder
+    from pip._internal.operations.build.build_tracker import BuildTracker
     from pip._internal.req.req_install import InstallRequirement
+    from pip._internal.resolution.base import BaseResolver
 
     class ExtraEnviron(TypedDict, total=False):
         extra_environ: dict[str, str]
@@ -245,6 +257,177 @@ def install(
             )
 
 
+class InprocessBuildEnvironmentInstaller:
+    """
+    Build dependency installer that runs in the same pip process.
+
+    This contains a stripped down version of the install command with
+    only the logic necessary for installing build dependencies. The
+    finder, session, build tracker, and wheel cache are reused, but new
+    instances of everything else are created as needed.
+
+    Options are inherited from the parent install command unless
+    they don't make sense for build dependencies (in which case, they
+    are hard-coded, see comments below).
+    """
+
+    def __init__(
+        self,
+        *,
+        finder: PackageFinder,
+        build_tracker: BuildTracker,
+        wheel_cache: WheelCache,
+        build_constraints: Sequence[InstallRequirement] = (),
+        verbosity: int = 0,
+    ) -> None:
+        from pip._internal.operations.prepare import RequirementPreparer
+
+        self._finder = finder
+        self._build_constraints = build_constraints
+        self._wheel_cache = wheel_cache
+        self._level = 0
+
+        build_dir = TempDirectory(kind="build-env-install", globally_managed=True)
+        self._preparer = RequirementPreparer(
+            build_isolation_installer=self,
+            # Inherited options or state.
+            finder=finder,
+            session=finder._link_collector.session,
+            build_dir=build_dir.path,
+            build_tracker=build_tracker,
+            verbosity=verbosity,
+            # This is irrelevant as it only applies to editable requirements.
+            src_dir="",
+            # Hard-coded options (that should NOT be inherited).
+            download_dir=None,
+            build_isolation=True,
+            check_build_deps=False,
+            progress_bar="off",
+            # TODO: hash-checking should be extended to build deps, but that is
+            # deferred for later as it'd be a breaking change.
+            require_hashes=False,
+            use_user_site=False,
+            lazy_wheel=False,
+            legacy_resolver=False,
+        )
+
+    def install(
+        self,
+        requirements: Iterable[str],
+        prefix: _Prefix,
+        *,
+        kind: str,
+        for_req: InstallRequirement | None,
+    ) -> None:
+        """Install entrypoint. Manages output capturing and error handling."""
+        capture_logs = not logger.isEnabledFor(VERBOSE) and self._level == 0
+        if capture_logs:
+            # Hide the logs from the installation of build dependencies.
+            # They will be shown only if an error occurs.
+            capture_ctx: ContextManager[StringIO] = capture_logging()
+            spinner: ContextManager[None] = open_rich_spinner(f"Installing {kind}")
+        else:
+            # Otherwise, pass-through all logs (with a header).
+            capture_ctx, spinner = nullcontext(StringIO()), nullcontext()
+            logger.info("Installing %s ...", kind)
+
+        try:
+            self._level += 1
+            with spinner, capture_ctx as stream:
+                self._install_impl(requirements, prefix)
+
+        except DiagnosticPipError as exc:
+            # Format similar to a nested subprocess error, where the
+            # causing error is shown first, followed by the build error.
+            logger.info(textwrap.dedent(stream.getvalue()))
+            logger.error("%s", exc, extra={"rich": True})
+            logger.info("")
+            raise BuildDependencyInstallError(
+                for_req, requirements, cause=exc, log_lines=None
+            )
+
+        except Exception as exc:
+            logs: list[str] | None = textwrap.dedent(stream.getvalue()).splitlines()
+            if not capture_logs:
+                # If logs aren't being captured, then display the error inline
+                # with the rest of the logs.
+                logs = None
+                if isinstance(exc, PipError):
+                    logger.error("%s", exc)
+                else:
+                    logger.exception("pip crashed unexpectedly")
+            raise BuildDependencyInstallError(
+                for_req, requirements, cause=exc, log_lines=logs
+            )
+
+        finally:
+            self._level -= 1
+
+    def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None:
+        """Core build dependency install logic."""
+        from pip._internal.commands.install import installed_packages_summary
+        from pip._internal.req import install_given_reqs
+        from pip._internal.req.constructors import install_req_from_line
+        from pip._internal.wheel_builder import build
+
+        ireqs = [install_req_from_line(req, user_supplied=True) for req in requirements]
+        ireqs.extend(self._build_constraints)
+
+        resolver = self._make_resolver()
+        resolved_set = resolver.resolve(ireqs, check_supported_wheels=True)
+        self._preparer.prepare_linked_requirements_more(
+            resolved_set.requirements.values()
+        )
+
+        reqs_to_build = [
+            r for r in resolved_set.requirements_to_install if not r.is_wheel
+        ]
+        _, build_failures = build(reqs_to_build, self._wheel_cache, verify=True)
+        if build_failures:
+            raise InstallWheelBuildError(build_failures)
+
+        installed = install_given_reqs(
+            resolver.get_installation_order(resolved_set),
+            prefix=prefix.path,
+            # Hard-coded options (that should NOT be inherited).
+            root=None,
+            home=None,
+            warn_script_location=False,
+            use_user_site=False,
+            # As the build environment is ephemeral, it's wasteful to
+            # pre-compile everything since not all modules will be used.
+            pycompile=False,
+            progress_bar="off",
+        )
+
+        env = get_environment(list(prefix.lib_dirs))
+        if summary := installed_packages_summary(installed, env):
+            logger.info(summary)
+
+    def _make_resolver(self) -> BaseResolver:
+        """Create a new resolver for one time use."""
+        # Legacy installer never used the legacy resolver so create a
+        # resolvelib resolver directly. Yuck.
+        from pip._internal.req.constructors import install_req_from_req_string
+        from pip._internal.resolution.resolvelib.resolver import Resolver
+
+        return Resolver(
+            make_install_req=install_req_from_req_string,
+            # Inherited state.
+            preparer=self._preparer,
+            finder=self._finder,
+            wheel_cache=self._wheel_cache,
+            # Hard-coded options (that should NOT be inherited).
+            ignore_requires_python=False,
+            use_user_site=False,
+            ignore_dependencies=False,
+            ignore_installed=True,
+            force_reinstall=False,
+            upgrade_strategy="to-satisfy-only",
+            py_version_info=None,
+        )
+
+
 class BuildEnvironment:
     """Creates and manages an isolated environment to install build deps"""
 

src/pip/_internal/cli/base_command.py

@@ -235,6 +235,17 @@ def _main(self, args: list[str]) -> int:
                 )
                 options.cache_dir = None
 
+        if (
+            "inprocess-build-deps" in options.features_enabled
+            and os.environ.get("PIP_CONSTRAINT", "")
+            and "build-constraint" not in options.features_enabled
+        ):
+            logger.warning(
+                "In-process build dependencies are enabled, "
+                "PIP_CONSTRAINT will have no effect for build dependencies"
+            )
+            options.features_enabled.append("build-constraint")
+
         return self._run_wrapper(level_number, options, args)
 
     def handler_map(self) -> dict[str, Callable[[Values, list[str]], None]]:

src/pip/_internal/cli/cmdoptions.py

@@ -1056,6 +1056,7 @@ def check_list_path_option(options: Values) -> None:
     choices=[
         "fast-deps",
         "build-constraint",
+        "inprocess-build-deps",
     ]
     + ALWAYS_ENABLED_FEATURES,
     help="Enable new functionality, that may be backward incompatible.",

src/pip/_internal/cli/req_command.py

@@ -13,7 +13,11 @@
 from optparse import Values
 from typing import Any, Callable, TypeVar
 
-from pip._internal.build_env import SubprocessBuildEnvironmentInstaller
+from pip._internal.build_env import (
+    BuildEnvironmentInstaller,
+    InprocessBuildEnvironmentInstaller,
+    SubprocessBuildEnvironmentInstaller,
+)
 from pip._internal.cache import WheelCache
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.cmdoptions import make_target_python
@@ -100,6 +104,31 @@ def wrapper(self: _CommandT, options: Values, args: list[str]) -> int:
     return wrapper
 
 
+def parse_constraint_files(
+    constraint_files: list[str],
+    finder: PackageFinder,
+    options: Values,
+    session: PipSession,
+) -> list[InstallRequirement]:
+    requirements = []
+    for filename in constraint_files:
+        for parsed_req in parse_requirements(
+            filename,
+            constraint=True,
+            finder=finder,
+            options=options,
+            session=session,
+        ):
+            req_to_add = install_req_from_parsed_requirement(
+                parsed_req,
+                isolated=options.isolated_mode,
+                user_supplied=False,
+            )
+            requirements.append(req_to_add)
+
+    return requirements
+
+
 class RequirementCommand(IndexGroupCommand):
     def __init__(self, *args: Any, **kw: Any) -> None:
         super().__init__(*args, **kw)
@@ -159,16 +188,31 @@ def make_requirement_preparer(
             "build-constraint" in options.features_enabled
         )
 
+        env_installer: BuildEnvironmentInstaller
+        if "inprocess-build-deps" in options.features_enabled:
+            build_constraint_reqs = parse_constraint_files(
+                build_constraints, finder, options, session
+            )
+            env_installer = InprocessBuildEnvironmentInstaller(
+                finder=finder,
+                build_tracker=build_tracker,
+                build_constraints=build_constraint_reqs,
+                verbosity=verbosity,
+                wheel_cache=WheelCache(options.cache_dir),
+            )
+        else:
+            env_installer = SubprocessBuildEnvironmentInstaller(
+                finder,
+                build_constraints=build_constraints,
+                build_constraint_feature_enabled=build_constraint_feature_enabled,
+            )
+
         return RequirementPreparer(
             build_dir=temp_build_dir_path,
             src_dir=options.src_dir,
             download_dir=download_dir,
             build_isolation=options.build_isolation,
-            build_isolation_installer=SubprocessBuildEnvironmentInstaller(
-                finder,
-                build_constraints=build_constraints,
-                build_constraint_feature_enabled=build_constraint_feature_enabled,
-            ),
+            build_isolation_installer=env_installer,
             check_build_deps=options.check_build_deps,
             build_tracker=build_tracker,
             session=session,
@@ -251,22 +295,14 @@ def get_requirements(
         requirements: list[InstallRequirement] = []
 
         if not should_ignore_regular_constraints(options):
-            for filename in options.constraints:
-                for parsed_req in parse_requirements(
-                    filename,
-                    constraint=True,
-                    finder=finder,
-                    options=options,
-                    session=session,
-                ):
-                    req_to_add = install_req_from_parsed_requirement(
-                        parsed_req,
-                        isolated=options.isolated_mode,
-                        user_supplied=False,
-                    )
-                    requirements.append(req_to_add)
+            constraints = parse_constraint_files(
+                options.constraints, finder, options, session
+            )
+            requirements.extend(constraints)
 
         for req in args:
+            if not req.strip():
+                continue
             req_to_add = install_req_from_line(
                 req,
                 comes_from=None,