From 1d7548d485bc22247df150f47cdf73a8eae450d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Thu, 14 Aug 2025 00:11:14 -0400 Subject: [PATCH 01/17] Merge pull request #2956 from esafak/refactor/2074-decouple-discovery --- docs/changelog/2074.feature.rst | 1 + src/virtualenv/create/via_global_ref/venv.py | 2 +- src/virtualenv/discovery/app_data.py | 23 + src/virtualenv/discovery/builtin.py | 31 +- src/virtualenv/discovery/cache.py | 18 + src/virtualenv/discovery/cached_py_info.py | 41 +- src/virtualenv/discovery/discover.py | 3 +- src/virtualenv/discovery/py_info.py | 28 +- src/virtualenv/discovery/windows/__init__.py | 10 +- src/virtualenv/run/__init__.py | 3 + src/virtualenv/run/plugin/discovery.py | 2 +- tests/__init__.py | 0 tests/conftest.py | 12 +- tests/integration/test_zipapp.py | 14 +- tests/unit/activation/conftest.py | 13 +- tests/unit/config/test_env_var.py | 7 +- tests/unit/create/conftest.py | 24 +- tests/unit/create/test_creator.py | 426 ++++++++++-------- tests/unit/create/test_interpreters.py | 24 +- .../create/via_global_ref/test_build_c_ext.py | 113 ++--- tests/unit/discovery/py_info/test_py_info.py | 68 +-- .../py_info/test_py_info_exe_based_of.py | 15 +- tests/unit/discovery/test_discovery.py | 79 ++-- tests/unit/discovery/util.py | 51 +++ tests/unit/discovery/windows/test_windows.py | 3 +- .../embed/test_bootstrap_link_via_app_data.py | 13 +- 26 files changed, 625 insertions(+), 399 deletions(-) create mode 100644 docs/changelog/2074.feature.rst create mode 100644 src/virtualenv/discovery/app_data.py create mode 100644 src/virtualenv/discovery/cache.py create mode 100644 tests/__init__.py create mode 100644 tests/unit/discovery/util.py diff --git a/docs/changelog/2074.feature.rst b/docs/changelog/2074.feature.rst new file mode 100644 index 000000000..61b1c38b5 --- /dev/null +++ b/docs/changelog/2074.feature.rst @@ -0,0 +1 @@ +Add AppData and Cache protocols to discovery for decoupling - by :user:`esafak`. diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py index d5037bc3b..45866d676 100644 --- a/src/virtualenv/create/via_global_ref/venv.py +++ b/src/virtualenv/create/via_global_ref/venv.py @@ -20,7 +20,7 @@ class Venv(ViaGlobalRefApi): def __init__(self, options, interpreter) -> None: self.describe = options.describe super().__init__(options, interpreter) - current = PythonInfo.current() + current = PythonInfo.current(options.app_data, options.cache) self.can_be_inline = interpreter is current and interpreter.executable == interpreter.system_executable self._context = None diff --git a/src/virtualenv/discovery/app_data.py b/src/virtualenv/discovery/app_data.py new file mode 100644 index 000000000..90de58284 --- /dev/null +++ b/src/virtualenv/discovery/app_data.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, ContextManager, Protocol + +if TYPE_CHECKING: + from pathlib import Path + + +class AppData(Protocol): + """Protocol for application data store.""" + + def py_info(self, path: Path) -> Any: ... + + def py_info_clear(self) -> None: ... + + @contextmanager + def ensure_extracted(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: ... + + @contextmanager + def extract(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: ... + + def close(self) -> None: ... diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index 6890213a4..3c6dcb2fd 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -18,7 +18,7 @@ from argparse import ArgumentParser from collections.abc import Callable, Generator, Iterable, Mapping, Sequence - from virtualenv.app_data.base import AppData + from .app_data import AppData LOGGER = logging.getLogger(__name__) @@ -27,8 +27,8 @@ class Builtin(Discover): app_data: AppData try_first_with: Sequence[str] - def __init__(self, options) -> None: - super().__init__(options) + def __init__(self, options, cache=None) -> None: + super().__init__(options, cache) self.python_spec = options.python or [sys.executable] if self._env.get("VIRTUALENV_PYTHON"): self.python_spec = self.python_spec[1:] + self.python_spec[:1] # Rotate the list @@ -60,7 +60,7 @@ def add_parser_arguments(cls, parser: ArgumentParser) -> None: def run(self) -> PythonInfo | None: for python_spec in self.python_spec: - result = get_interpreter(python_spec, self.try_first_with, self.app_data, self._env) + result = get_interpreter(python_spec, self.try_first_with, self.app_data, self.cache, self._env) if result is not None: return result return None @@ -71,13 +71,17 @@ def __repr__(self) -> str: def get_interpreter( - key, try_first_with: Iterable[str], app_data: AppData | None = None, env: Mapping[str, str] | None = None + key, + try_first_with: Iterable[str], + app_data: AppData | None = None, + cache=None, + env: Mapping[str, str] | None = None, ) -> PythonInfo | None: spec = PythonSpec.from_string_spec(key) LOGGER.info("find interpreter for spec %r", spec) proposed_paths = set() env = os.environ if env is None else env - for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env): + for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, cache, env): key = interpreter.system_executable, impl_must_match if key in proposed_paths: continue @@ -93,6 +97,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 spec: PythonSpec, try_first_with: Iterable[str], app_data: AppData | None = None, + cache=None, env: Mapping[str, str] | None = None, ) -> Generator[tuple[PythonInfo, bool], None, None]: # 0. if it's a path and exists, and is absolute path, this is the only option we consider @@ -108,7 +113,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 exe_id = fs_path_id(exe_raw) if exe_id not in tested_exes: tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, env=env), True + yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True return # 1. try with first @@ -124,7 +129,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 if exe_id in tested_exes: continue tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, env=env), True + yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True # 1. if it's a path and exists if spec.path is not None: @@ -137,12 +142,12 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 exe_id = fs_path_id(exe_raw) if exe_id not in tested_exes: tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, env=env), True + yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True if spec.is_abs: return else: # 2. otherwise try with the current - current_python = PythonInfo.current_system(app_data) + current_python = PythonInfo.current_system(app_data, cache) exe_raw = str(current_python.executable) exe_id = fs_path_id(exe_raw) if exe_id not in tested_exes: @@ -153,7 +158,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 if IS_WIN: from .windows import propose_interpreters # noqa: PLC0415 - for interpreter in propose_interpreters(spec, app_data, env): + for interpreter in propose_interpreters(spec, app_data, cache, env): exe_raw = str(interpreter.executable) exe_id = fs_path_id(exe_raw) if exe_id in tested_exes: @@ -171,7 +176,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 if exe_id in tested_exes: continue tested_exes.add(exe_id) - interpreter = PathPythonInfo.from_exe(exe_raw, app_data, raise_on_error=False, env=env) + interpreter = PathPythonInfo.from_exe(exe_raw, app_data, cache, raise_on_error=False, env=env) if interpreter is not None: yield interpreter, impl_must_match @@ -184,7 +189,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 uv_python_path = user_data_path("uv") / "python" for exe_path in uv_python_path.glob("*/bin/python"): - interpreter = PathPythonInfo.from_exe(str(exe_path), app_data, raise_on_error=False, env=env) + interpreter = PathPythonInfo.from_exe(str(exe_path), app_data, cache, raise_on_error=False, env=env) if interpreter is not None: yield interpreter, True diff --git a/src/virtualenv/discovery/cache.py b/src/virtualenv/discovery/cache.py new file mode 100644 index 000000000..eaf24cc14 --- /dev/null +++ b/src/virtualenv/discovery/cache.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from pathlib import Path + + +class Cache(Protocol): + """A protocol for a cache.""" + + def get(self, path: Path) -> Any: ... + + def set(self, path: Path, data: Any) -> None: ... + + def remove(self, path: Path) -> None: ... + + def clear(self) -> None: ... diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index f0a1dc609..645a5eb36 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -20,13 +20,11 @@ from string import ascii_lowercase, ascii_uppercase, digits from typing import TYPE_CHECKING -from virtualenv.app_data.na import AppDataDisabled -from virtualenv.cache import FileCache +from .py_info import PythonInfo if TYPE_CHECKING: - from virtualenv.app_data.base import AppData - from virtualenv.cache import Cache -from virtualenv.discovery.py_info import PythonInfo + from .app_data import AppData + from .cache import Cache _CACHE = OrderedDict() _CACHE[Path(sys.executable)] = PythonInfo() @@ -35,19 +33,15 @@ def from_exe( # noqa: PLR0913 cls, - app_data, - exe, - env=None, + app_data: AppData, + exe: str, + env: dict[str, str] | None = None, *, - raise_on_error=True, - ignore_cache=False, - cache: Cache | None = None, + raise_on_error: bool = True, + ignore_cache: bool = False, + cache: Cache, ) -> PythonInfo | None: env = os.environ if env is None else env - if cache is None: - if app_data is None: - app_data = AppDataDisabled() - cache = FileCache(store_factory=app_data.py_info, clearer=app_data.py_info_clear) result = _get_from_cache(cls, app_data, exe, env, cache, ignore_cache=ignore_cache) if isinstance(result, Exception): if raise_on_error: @@ -123,7 +117,12 @@ def gen_cookie(): ) -def _run_subprocess(cls, exe, app_data, env): +def _run_subprocess( + cls, + exe: str, + app_data: AppData, + env: dict[str, str], +) -> tuple[Exception | None, PythonInfo | None]: py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" # Cookies allow to split the serialized stdout output generated by the script collecting the info from the output # generated by something else. The right way to deal with it is to create an anonymous pipe and pass its descriptor @@ -135,10 +134,8 @@ def _run_subprocess(cls, exe, app_data, env): start_cookie = gen_cookie() end_cookie = gen_cookie() - if app_data is None: - app_data = AppDataDisabled() - with app_data.ensure_extracted(py_info_script) as py_info_script: - cmd = [exe, str(py_info_script), start_cookie, end_cookie] + with app_data.ensure_extracted(py_info_script) as py_info_script_path: + cmd = [exe, str(py_info_script_path), start_cookie, end_cookie] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = env.copy() env.pop("__PYVENV_LAUNCHER__", None) @@ -199,10 +196,8 @@ def __repr__(self) -> str: return cmd_repr -def clear(app_data=None, cache=None): +def clear(cache: Cache | None = None) -> None: """Clear the cache.""" - if cache is None and app_data is not None: - cache = FileCache(store_factory=app_data.py_info, clearer=app_data.py_info_clear) if cache is not None: cache.clear() _CACHE.clear() diff --git a/src/virtualenv/discovery/discover.py b/src/virtualenv/discovery/discover.py index 0aaa17c8e..de1b5fd0b 100644 --- a/src/virtualenv/discovery/discover.py +++ b/src/virtualenv/discovery/discover.py @@ -15,7 +15,7 @@ def add_parser_arguments(cls, parser): """ raise NotImplementedError - def __init__(self, options) -> None: + def __init__(self, options, cache=None) -> None: """ Create a new discovery mechanism. @@ -24,6 +24,7 @@ def __init__(self, options) -> None: self._has_run = False self._interpreter = None self._env = options.env + self.cache = cache @abstractmethod def run(self): diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 5f16dbc8a..797f88bdb 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -378,11 +378,11 @@ def spec(self): ) @classmethod - def clear_cache(cls, app_data, cache=None): + def clear_cache(cls, cache=None): # this method is not used by itself, so here and called functions can import stuff locally from virtualenv.discovery.cached_py_info import clear # noqa: PLC0415 - clear(app_data, cache) + clear(cache) cls._cache_exe_discovery.clear() def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 @@ -423,7 +423,7 @@ def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 _current = None @classmethod - def current(cls, app_data=None, cache=None): + def current(cls, app_data, cache): """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. @@ -432,14 +432,14 @@ def current(cls, app_data=None, cache=None): cls._current = cls.from_exe( sys.executable, app_data, + cache, raise_on_error=True, resolve_to_host=False, - cache=cache, ) return cls._current @classmethod - def current_system(cls, app_data=None, cache=None) -> PythonInfo: + def current_system(cls, app_data, cache) -> PythonInfo: """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. @@ -448,9 +448,9 @@ def current_system(cls, app_data=None, cache=None) -> PythonInfo: cls._current_system = cls.from_exe( sys.executable, app_data, + cache, raise_on_error=True, resolve_to_host=True, - cache=cache, ) return cls._current_system @@ -467,12 +467,12 @@ def _to_dict(self): def from_exe( # noqa: PLR0913 cls, exe, - app_data=None, + app_data, + cache, raise_on_error=True, # noqa: FBT002 ignore_cache=False, # noqa: FBT002 resolve_to_host=True, # noqa: FBT002 env=None, - cache=None, ): """Given a path to an executable get the python information.""" # this method is not used by itself, so here and called functions can import stuff locally @@ -513,7 +513,7 @@ def _from_dict(cls, data): return result @classmethod - def _resolve_to_system(cls, app_data, target, cache=None): + def _resolve_to_system(cls, app_data, target, cache): start_executable = target.executable prefixes = OrderedDict() while target.system_executable is None: @@ -532,13 +532,13 @@ def _resolve_to_system(cls, app_data, target, cache=None): prefixes[prefix] = target target = target.discover_exe(app_data, prefix=prefix, exact=False, cache=cache) if target.executable != target.system_executable: - target = cls.from_exe(target.system_executable, app_data, cache=cache) + target = cls.from_exe(target.system_executable, app_data, cache) target.executable = start_executable return target _cache_exe_discovery = {} # noqa: RUF012 - def discover_exe(self, app_data, prefix, exact=True, env=None, cache=None): # noqa: FBT002 + def discover_exe(self, app_data, cache, prefix, exact=True, env=None): # noqa: FBT002 key = prefix, exact if key in self._cache_exe_discovery and prefix: LOGGER.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) @@ -551,7 +551,7 @@ def discover_exe(self, app_data, prefix, exact=True, env=None, cache=None): # n env = os.environ if env is None else env for folder in possible_folders: for name in possible_names: - info = self._check_exe(app_data, folder, name, exact, discovered, env, cache) + info = self._check_exe(app_data, cache, folder, name, exact, discovered, env) if info is not None: self._cache_exe_discovery[key] = info return info @@ -564,17 +564,17 @@ def discover_exe(self, app_data, prefix, exact=True, env=None, cache=None): # n msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) raise RuntimeError(msg) - def _check_exe(self, app_data, folder, name, exact, discovered, env, cache): # noqa: PLR0913 + def _check_exe(self, app_data, cache, folder, name, exact, discovered, env): # noqa: PLR0913 exe_path = os.path.join(folder, name) if not os.path.exists(exe_path): return None info = self.from_exe( exe_path, app_data, + cache, resolve_to_host=False, raise_on_error=False, env=env, - cache=cache, ) if info is None: # ignore if for some reason we can't query return None diff --git a/src/virtualenv/discovery/windows/__init__.py b/src/virtualenv/discovery/windows/__init__.py index b7206406a..ef47a90e3 100644 --- a/src/virtualenv/discovery/windows/__init__.py +++ b/src/virtualenv/discovery/windows/__init__.py @@ -16,7 +16,7 @@ class Pep514PythonInfo(PythonInfo): """A Python information acquired from PEP-514.""" -def propose_interpreters(spec, cache_dir, env): +def propose_interpreters(spec, app_data, cache, env): # see if PEP-514 entries are good # start with higher python versions in an effort to use the latest version available @@ -36,7 +36,13 @@ def propose_interpreters(spec, cache_dir, env): skip_pre_filter = implementation.lower() != "cpython" registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe, free_threaded=threaded) if skip_pre_filter or registry_spec.satisfies(spec): - interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False) + interpreter = Pep514PythonInfo.from_exe( + exe, + app_data, + cache, + raise_on_error=False, + env=env, + ) if interpreter is not None and interpreter.satisfies(spec, impl_must_match=True): yield interpreter # Final filtering/matching using interpreter metadata diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index 03190502b..208050c90 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -5,6 +5,7 @@ from functools import partial from virtualenv.app_data import make_app_data +from virtualenv.cache import FileCache from virtualenv.config.cli.parser import VirtualEnvConfigParser from virtualenv.report import LEVELS, setup_report from virtualenv.run.session import Session @@ -130,6 +131,8 @@ def load_app_data(args, parser, options): options, _ = parser.parse_known_args(args, namespace=options) if options.reset_app_data: options.app_data.reset() + + options.cache = FileCache(store_factory=options.app_data.py_info, clearer=options.app_data.py_info_clear) return options diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py index 5e8b2392f..b271faac5 100644 --- a/src/virtualenv/run/plugin/discovery.py +++ b/src/virtualenv/run/plugin/discovery.py @@ -32,7 +32,7 @@ def get_discover(parser, args): discover_class = discover_types[options.discovery] discover_class.add_parser_arguments(discovery_parser) options, _ = parser.parse_known_args(args, namespace=options) - return discover_class(options) + return discover_class(options, options.cache) def _get_default_discovery(discover_types): diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/conftest.py b/tests/conftest.py index dbf8d0edf..4fcb25da8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ import pytest from virtualenv.app_data import AppDataDiskFolder +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_GRAALPY, IS_PYPY, IS_WIN, fs_supports_symlink from virtualenv.report import LOGGER @@ -125,9 +126,10 @@ def _check_cwd_not_changed_by_test(): @pytest.fixture(autouse=True) def _ensure_py_info_cache_empty(session_app_data): - PythonInfo.clear_cache(session_app_data) + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + PythonInfo.clear_cache(cache) yield - PythonInfo.clear_cache(session_app_data) + PythonInfo.clear_cache(cache) @contextmanager @@ -309,7 +311,8 @@ def special_name_dir(tmp_path, special_char_name): @pytest.fixture(scope="session") def current_creators(session_app_data): - return CreatorSelector.for_interpreter(PythonInfo.current_system(session_app_data)) + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return CreatorSelector.for_interpreter(PythonInfo.current_system(session_app_data, cache)) @pytest.fixture(scope="session") @@ -357,7 +360,8 @@ def for_py_version(): @pytest.fixture def _skip_if_test_in_system(session_app_data): - current = PythonInfo.current(session_app_data) + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + current = PythonInfo.current(session_app_data, cache) if current.system_executable is not None: pytest.skip("test not valid if run under system") diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py index c6c46c7bc..e40ecca91 100644 --- a/tests/integration/test_zipapp.py +++ b/tests/integration/test_zipapp.py @@ -7,19 +7,25 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink from virtualenv.run import cli_run HERE = Path(__file__).parent -CURRENT = PythonInfo.current_system() @pytest.fixture(scope="session") -def zipapp_build_env(tmp_path_factory): +def current_info(session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return PythonInfo.current_system(session_app_data, cache) + + +@pytest.fixture(scope="session") +def zipapp_build_env(tmp_path_factory, current_info): create_env_path = None - if CURRENT.implementation not in {"PyPy", "GraalVM"}: - exe = CURRENT.executable # guaranteed to contain a recent enough pip (tox.ini) + if current_info.implementation not in {"PyPy", "GraalVM"}: + exe = current_info.executable # guaranteed to contain a recent enough pip (tox.ini) else: create_env_path = tmp_path_factory.mktemp("zipapp-create-env") exe, found = None, False diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 53a819f96..a7186896f 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -229,9 +229,18 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session", params=[True, False], ids=["with_prompt", "no_prompt"]) -def activation_python(request, tmp_path_factory, special_char_name, current_fastest): +def activation_python(request, tmp_path_factory, special_char_name, current_fastest, session_app_data): dest = os.path.join(str(tmp_path_factory.mktemp("activation-tester-env")), special_char_name) - cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv", "--no-periodic-update"] + cmd = [ + "--without-pip", + dest, + "--creator", + current_fastest, + "-vv", + "--no-periodic-update", + "--app-data", + str(session_app_data), + ] # `params` is accessed here. https://docs.pytest.org/en/stable/reference/reference.html#pytest-fixture if request.param: cmd += ["--prompt", special_char_name] diff --git a/tests/unit/config/test_env_var.py b/tests/unit/config/test_env_var.py index 5f364978d..8ec16c607 100644 --- a/tests/unit/config/test_env_var.py +++ b/tests/unit/config/test_env_var.py @@ -5,6 +5,7 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.config.cli.parser import VirtualEnvOptions from virtualenv.config.ini import IniConfig from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew @@ -76,10 +77,12 @@ def test_extra_search_dir_via_env_var(tmp_path, monkeypatch): @pytest.mark.usefixtures("_empty_conf") -@pytest.mark.skipif(is_macos_brew(PythonInfo.current_system()), reason="no copy on brew") -def test_value_alias(monkeypatch, mocker): +def test_value_alias(monkeypatch, mocker, session_app_data): from virtualenv.config.cli.parser import VirtualEnvConfigParser # noqa: PLC0415 + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + if is_macos_brew(PythonInfo.current_system(session_app_data, cache)): + pytest.skip(reason="no copy on brew") prev = VirtualEnvConfigParser._fix_default # noqa: SLF001 def func(self, action): diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py index 58d390c5c..4c7bf45c0 100644 --- a/tests/unit/create/conftest.py +++ b/tests/unit/create/conftest.py @@ -14,25 +14,31 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo -CURRENT = PythonInfo.current_system() +@pytest.fixture(scope="session") +def current_info(session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return PythonInfo.current_system(session_app_data, cache) -def root(tmp_path_factory, session_app_data): # noqa: ARG001 - return CURRENT.system_executable +def root(tmp_path_factory, session_app_data, current_info): # noqa: ARG001 + return current_info.system_executable -def venv(tmp_path_factory, session_app_data): - if CURRENT.is_venv: + +def venv(tmp_path_factory, session_app_data, current_info): + if current_info.is_venv: return sys.executable - root_python = root(tmp_path_factory, session_app_data) + root_python = root(tmp_path_factory, session_app_data, current_info) dest = tmp_path_factory.mktemp("venv") process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)]) process.communicate() # sadly creating a virtual environment does not tell us where the executable lives in general case # so discover using some heuristic - return CURRENT.discover_exe(prefix=str(dest)).original_executable + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return current_info.discover_exe(session_app_data, cache, prefix=str(dest)).original_executable PYTHON = { @@ -42,8 +48,8 @@ def venv(tmp_path_factory, session_app_data): @pytest.fixture(params=list(PYTHON.values()), ids=list(PYTHON.keys()), scope="session") -def python(request, tmp_path_factory, session_app_data): - result = request.param(tmp_path_factory, session_app_data) +def python(request, tmp_path_factory, session_app_data, current_info): + result = request.param(tmp_path_factory, session_app_data, current_info) if isinstance(result, Exception): pytest.skip(f"could not resolve interpreter based on {request.param.__name__} because {result}") if result is None: diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index 910f41e13..0e80514b1 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -14,7 +14,6 @@ import textwrap import zipfile from collections import OrderedDict -from itertools import product from pathlib import Path from stat import S_IREAD, S_IRGRP, S_IROTH from textwrap import dedent @@ -23,6 +22,7 @@ import pytest from virtualenv.__main__ import run, run_with_catch +from virtualenv.cache import FileCache from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info from virtualenv.create.pyenv_cfg import PyEnvCfg from virtualenv.create.via_global_ref import api @@ -33,7 +33,13 @@ from virtualenv.run import cli_run, session_via_cli from virtualenv.run.plugin.creators import CreatorSelector -CURRENT = PythonInfo.current_system() +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="session") +def current_info(session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return PythonInfo.current_system(session_app_data, cache) def test_os_path_sep_not_allowed(tmp_path, capsys): @@ -90,140 +96,179 @@ def cleanup_sys_path(paths): @pytest.fixture(scope="session") -def system(session_app_data): - return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) +def system(session_app_data, current_info): + return get_env_debug_info(Path(current_info.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) -CURRENT_CREATORS = [i for i in CreatorSelector.for_interpreter(CURRENT).key_to_class if i != "builtin"] -CREATE_METHODS = [] -for k, v in CreatorSelector.for_interpreter(CURRENT).key_to_meta.items(): - if k in CURRENT_CREATORS: - if v.can_copy: - if k == "venv" and CURRENT.implementation == "PyPy" and CURRENT.pypy_version_info >= [7, 3, 13]: - continue # https://foss.heptapod.net/pypy/pypy/-/issues/4019 - CREATE_METHODS.append((k, "copies")) - if v.can_symlink: - CREATE_METHODS.append((k, "symlinks")) +@pytest.fixture(scope="session") +def current_creator_keys(current_info): + return [i for i in CreatorSelector.for_interpreter(current_info).key_to_class if i != "builtin"] -@pytest.mark.parametrize( - ("creator", "isolated"), - [pytest.param(*i, id=f"{'-'.join(i[0])}-{i[1]}") for i in product(CREATE_METHODS, ["isolated", "global"])], -) +@pytest.fixture(scope="session") +def create_methods(current_creator_keys, current_info): + methods = [] + for k, v in CreatorSelector.for_interpreter(current_info).key_to_meta.items(): + if k in current_creator_keys: + if v.can_copy: + if ( + k == "venv" + and current_info.implementation == "PyPy" + and current_info.pypy_version_info >= [7, 3, 13] + ): # https://github.com/pypy/pypy/issues/4019 + continue + methods.append((k, "copies")) + if v.can_symlink: + methods.append((k, "symlinks")) + return methods + + +@pytest.fixture +def python_case(request, current_info): + """Resolve the python under test based on a param value.""" + case = request.param + if case == "venv": + # keep the original skip condition + if sys.executable == current_info.system_executable: + pytest.skip("system") + return sys.executable, "venv" + if case == "root": + return current_info.system_executable, "root" + msg = f"unknown python_case: {case}" + raise RuntimeError(msg) + + +@pytest.mark.parametrize("isolated", ["isolated", "global"]) +@pytest.mark.parametrize("python_case", ["venv", "root"], indirect=True) def test_create_no_seed( # noqa: C901, PLR0912, PLR0913, PLR0915 - python, - creator, - isolated, system, coverage_env, special_name_dir, + create_methods, + current_info, + session_app_data, + isolated, + python_case, ): - dest = special_name_dir - creator_key, method = creator - cmd = [ - "-v", - "-v", - "-p", - str(python), - str(dest), - "--without-pip", - "--activators", - "", - "--creator", - creator_key, - f"--{method}", - ] - if isolated == "global": - cmd.append("--system-site-packages") - result = cli_run(cmd) - creator = result.creator - coverage_env() - if IS_PYPY: - # pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits - # force a close of these on system where the limit is low-ish (e.g. MacOS 256) - gc.collect() - purelib = creator.purelib - patch_files = {purelib / f"{'_virtualenv'}.{i}" for i in ("py", "pyc", "pth")} - patch_files.add(purelib / "__pycache__") - content = set(creator.purelib.iterdir()) - patch_files - assert not content, "\n".join(str(i) for i in content) - assert creator.env_name == str(dest.name) - debug = creator.debug - assert "exception" not in debug, f"{debug.get('exception')}\n{debug.get('out')}\n{debug.get('err')}" - sys_path = cleanup_sys_path(debug["sys"]["path"]) - system_sys_path = cleanup_sys_path(system["sys"]["path"]) - our_paths = set(sys_path) - set(system_sys_path) - our_paths_repr = "\n".join(repr(i) for i in our_paths) - - # ensure we have at least one extra path added - assert len(our_paths) >= 1, our_paths_repr - # ensure all additional paths are related to the virtual environment - for path in our_paths: - msg = "\n".join(str(p) for p in system_sys_path) - msg = f"\n{path!s}\ndoes not start with {dest!s}\nhas:\n{msg}" - assert str(path).startswith(str(dest)), msg - # ensure there's at least a site-packages folder as part of the virtual environment added - assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr - - # ensure the global site package is added or not, depending on flag - global_sys_path = system_sys_path[-1] - if isolated == "isolated": - msg = "\n".join(str(j) for j in sys_path) - msg = f"global sys path {global_sys_path!s} is in virtual environment sys path:\n{msg}" - assert global_sys_path not in sys_path, msg - else: - common = [] - for left, right in zip(reversed(system_sys_path), reversed(sys_path)): - if left == right: - common.append(left) - else: - break - - def list_to_str(iterable): - return [str(i) for i in iterable] - - assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path))) - - # test that the python executables in the bin directory are either: - # - files - # - absolute symlinks outside of the venv - # - relative symlinks inside of the venv - if sys.platform == "win32": - exes = ("python.exe",) - else: - exes = ("python", f"python{sys.version_info.major}", f"python{sys.version_info.major}.{sys.version_info.minor}") - if creator_key == "venv": - # for venv some repackaging does not includes the pythonx.y - exes = exes[:-1] - for exe in exes: - exe_path = creator.bin_dir / exe - assert exe_path.exists(), "\n".join(str(i) for i in creator.bin_dir.iterdir()) - if not exe_path.is_symlink(): # option 1: a real file - continue # it was a file - link = os.readlink(str(exe_path)) - if not os.path.isabs(link): # option 2: a relative symlink - continue - # option 3: an absolute symlink, should point outside the venv - assert not link.startswith(str(creator.dest)) - - if IS_WIN and CURRENT.implementation == "CPython": - python_w = creator.exe.parent / "pythonw.exe" - assert python_w.exists() - assert python_w.read_bytes() != creator.exe.read_bytes() - - if CPython3Posix.pyvenv_launch_patch_active(PythonInfo.from_exe(python)) and creator_key != "venv": - result = subprocess.check_output( - [str(creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'], - text=True, - ).strip() - assert result == "None" - - git_ignore = (dest / ".gitignore").read_text(encoding="utf-8") - if creator_key == "venv" and sys.version_info >= (3, 13): - comment = "# Created by venv; see https://docs.python.org/3/library/venv.html" - else: - comment = "# created by virtualenv automatically" - assert git_ignore.splitlines() == [comment, "*"] + python_exe, python_id = python_case + logger.info("running no seed test for %s-%s", python_id, isolated) + + for creator_key, method in create_methods: + dest = special_name_dir / f"{creator_key}-{method}-{isolated}" + cmd = [ + "-v", + "-v", + "-p", + str(python_exe), + str(dest), + "--without-pip", + "--activators", + "", + "--creator", + creator_key, + f"--{method}", + ] + if isolated == "global": + cmd.append("--system-site-packages") + result = cli_run(cmd) + creator = result.creator + coverage_env() + if IS_PYPY: + # pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits + # force a close of these on system where the limit is low-ish (e.g. MacOS 256) + gc.collect() + purelib = creator.purelib + patch_files = {purelib / f"{'_virtualenv'}.{i}" for i in ("py", "pyc", "pth")} + patch_files.add(purelib / "__pycache__") + content = set(creator.purelib.iterdir()) - patch_files + assert not content, "\n".join(str(i) for i in content) + assert creator.env_name == str(dest.name) + debug = creator.debug + assert "exception" not in debug, f"{debug.get('exception')}\n{debug.get('out')}\n{debug.get('err')}" + sys_path = cleanup_sys_path(debug["sys"]["path"]) + system_sys_path = cleanup_sys_path(system["sys"]["path"]) + our_paths = set(sys_path) - set(system_sys_path) + our_paths_repr = "\n".join(repr(i) for i in our_paths) + + # ensure we have at least one extra path added + assert len(our_paths) >= 1, our_paths_repr + # ensure all additional paths are related to the virtual environment + for path in our_paths: + msg = "\n".join(str(p) for p in system_sys_path) + msg = f"\n{path!s}\ndoes not start with {dest!s}\nhas:\n{msg}" + assert str(path).startswith(str(dest)), msg + # ensure there's at least a site-packages folder as part of the virtual environment added + assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr + + # ensure the global site package is added or not, depending on flag + global_sys_path = system_sys_path[-1] + if isolated == "isolated": + msg = "\n".join(str(j) for j in sys_path) + msg = f"global sys path {global_sys_path!s} is in virtual environment sys path:\n{msg}" + assert global_sys_path not in sys_path, msg + else: + common = [] + for left, right in zip(reversed(system_sys_path), reversed(sys_path)): + if left == right: + common.append(left) + else: + break + + def list_to_str(iterable): + return [str(i) for i in iterable] + + assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path))) + + # test that the python executables in the bin directory are either: + # - files + # - absolute symlinks outside of the venv + # - relative symlinks inside of the venv + if sys.platform == "win32": + exes = ("python.exe",) + else: + exes = ( + "python", + f"python{sys.version_info.major}", + f"python{sys.version_info.major}.{sys.version_info.minor}", + ) + if creator_key == "venv": + # for venv some repackaging does not includes the pythonx.y + exes = exes[:-1] + for exe in exes: + exe_path = creator.bin_dir / exe + assert exe_path.exists(), "\n".join(str(i) for i in creator.bin_dir.iterdir()) + if not exe_path.is_symlink(): # option 1: a real file + continue # it was a file + link = os.readlink(str(exe_path)) + if not os.path.isabs(link): # option 2: a relative symlink + continue + # option 3: an absolute symlink, should point outside the venv + assert not link.startswith(str(creator.dest)) + + if IS_WIN and current_info.implementation == "CPython": + python_w = creator.exe.parent / "pythonw.exe" + assert python_w.exists() + assert python_w.read_bytes() != creator.exe.read_bytes() + + if creator_key != "venv" and CPython3Posix.pyvenv_launch_patch_active( + PythonInfo.from_exe( + python_exe, + session_app_data, + FileCache(session_app_data.py_info, session_app_data.py_info_clear), + ), + ): + result = subprocess.check_output( + [str(creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'], + text=True, + ).strip() + assert result == "None" + + git_ignore = (dest / ".gitignore").read_text(encoding="utf-8") + if creator_key == "venv" and sys.version_info >= (3, 13): + comment = "# Created by venv; see https://docs.python.org/3/library/venv.html" + else: + comment = "# created by virtualenv automatically" + assert git_ignore.splitlines() == [comment, "*"] def test_create_cachedir_tag(tmp_path): @@ -273,8 +318,9 @@ def test_create_vcs_ignore_exists_override(tmp_path): assert git_ignore.read_text(encoding="utf-8") == "magic" -@pytest.mark.skipif(not CURRENT.has_venv, reason="requires interpreter with venv") -def test_venv_fails_not_inline(tmp_path, capsys, mocker): +def test_venv_fails_not_inline(tmp_path, capsys, mocker, current_info): + if not current_info.has_venv: + pytest.skip("requires interpreter with venv") if hasattr(os, "geteuid") and os.geteuid() == 0: pytest.skip("no way to check permission restriction when running under root") @@ -290,7 +336,7 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): cfg = str(cfg_path) try: os.chmod(cfg, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH) - cmd = ["-p", str(CURRENT.executable), str(tmp_path), "--without-pip", "--creator", "venv"] + cmd = ["-p", str(current_info.executable), str(tmp_path), "--without-pip", "--creator", "venv"] with pytest.raises(SystemExit) as context: run(cmd) assert context.value.code != 0 @@ -301,46 +347,45 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): assert "Error:" in err, err -@pytest.mark.parametrize("creator", CURRENT_CREATORS) @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"]) -def test_create_clear_resets(tmp_path, creator, clear, caplog): +def test_create_clear_resets(tmp_path, clear, caplog, current_creator_keys): caplog.set_level(logging.DEBUG) - if creator == "venv" and clear is False: - pytest.skip("venv without clear might fail") - marker = tmp_path / "magic" - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator, "-vvv"] - cli_run(cmd) + for creator in current_creator_keys: + if creator == "venv" and clear is False: + pytest.skip("venv without clear might fail") + marker = tmp_path / creator / "magic" + cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator, "-vvv"] + cli_run(cmd) - marker.write_text("", encoding="utf-8") # if we a marker file this should be gone on a clear run, remain otherwise - assert marker.exists() + marker.write_text("", encoding="utf-8") + assert marker.exists() - cli_run(cmd + (["--clear"] if clear else [])) - assert marker.exists() is not clear + cli_run(cmd + (["--clear"] if clear else [])) + assert marker.exists() is not clear -@pytest.mark.parametrize("creator", CURRENT_CREATORS) @pytest.mark.parametrize("prompt", [None, "magic"]) -def test_prompt_set(tmp_path, creator, prompt): - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] - if prompt is not None: - cmd.extend(["--prompt", "magic"]) - - result = cli_run(cmd) - actual_prompt = tmp_path.name if prompt is None else prompt - cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) - if prompt is None: - assert "prompt" not in cfg - elif creator != "venv": - assert "prompt" in cfg, list(cfg.content.keys()) - assert cfg["prompt"] == actual_prompt - - -@pytest.mark.parametrize("creator", CURRENT_CREATORS) -def test_home_path_is_exe_parent(tmp_path, creator): - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] - - result = cli_run(cmd) - cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) +def test_prompt_set(tmp_path, prompt, current_creator_keys): + for creator in current_creator_keys: + cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator] + if prompt is not None: + cmd.extend(["--prompt", "magic"]) + + result = cli_run(cmd) + actual_prompt = tmp_path.name if prompt is None else prompt + cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) + if prompt is None: + assert "prompt" not in cfg + elif creator != "venv": + assert "prompt" in cfg, list(cfg.content.keys()) + assert cfg["prompt"] == actual_prompt + + +def test_home_path_is_exe_parent(tmp_path, current_creator_keys): + for creator in current_creator_keys: + cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator] + result = cli_run(cmd) + cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) # Cannot assume "home" path is a specific value as path resolution may change # between versions (symlinks, framework paths, etc) but we can check that a @@ -401,25 +446,20 @@ def test_create_long_path(tmp_path): @pytest.mark.slow -@pytest.mark.parametrize( - "creator", - sorted(set(CreatorSelector.for_interpreter(PythonInfo.current_system()).key_to_class) - {"builtin"}), -) -@pytest.mark.usefixtures("session_app_data") -def test_create_distutils_cfg(creator, tmp_path, monkeypatch): - result = cli_run( - [ - str(tmp_path / "venv"), - "--activators", - "", - "--creator", - creator, - "--setuptools", - "bundle", - ], - ) - - app = Path(__file__).parent / "console_app" +def test_create_distutils_cfg(tmp_path, monkeypatch, current_creator_keys): + for creator in current_creator_keys: + result = cli_run( + [ + str(tmp_path / creator / "venv"), + "--activators", + "", + "--creator", + creator, + "--setuptools", + "bundle", + ], + ) + app = Path(__file__).parent / "console_app" dest = tmp_path / "console_app" shutil.copytree(str(app), str(dest)) @@ -468,9 +508,10 @@ def list_files(path): return result -@pytest.mark.skipif(is_macos_brew(CURRENT), reason="no copy on brew") @pytest.mark.skip(reason="https://github.com/pypa/setuptools/issues/4640") -def test_zip_importer_can_import_setuptools(tmp_path): +def test_zip_importer_can_import_setuptools(tmp_path, current_info): + if is_macos_brew(current_info): + pytest.skip("no copy on brew") """We're patching the loaders so might fail on r/o loaders, such as zipimporter on CPython<3.8""" result = cli_run( [str(tmp_path / "venv"), "--activators", "", "--no-pip", "--no-wheel", "--copies", "--setuptools", "bundle"], @@ -679,8 +720,9 @@ def _get_sys_path(flag=None): # (specifically venv scripts delivered with Python itself) are not writable. # # https://github.com/pypa/virtualenv/issues/2419 -@pytest.mark.skipif("venv" not in CURRENT_CREATORS, reason="test needs venv creator") -def test_venv_creator_without_write_perms(tmp_path, mocker): +def test_venv_creator_without_write_perms(tmp_path, mocker, current_creator_keys): + if "venv" not in current_creator_keys: + pytest.skip("test needs venv creator") from virtualenv.run.session import Session # noqa: PLC0415 prev = Session._create # noqa: SLF001 @@ -697,9 +739,10 @@ def func(self): cli_run(cmd) -def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker): +def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker, session_app_data): """Test that creating a virtual environment falls back to copies when filesystem has no symlink support.""" - if is_macos_brew(PythonInfo.from_exe(python)): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + if is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): pytest.skip("brew python on darwin may not support copies, which is tested separately") # Given a filesystem that does not support symlinks @@ -722,13 +765,14 @@ def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker): assert result.creator.symlinks is False -def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker): +def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker, session_app_data): """Test that virtualenv fails gracefully when no creation method is supported.""" # Given a filesystem that does not support symlinks mocker.patch("virtualenv.create.via_global_ref.api.fs_supports_symlink", return_value=False) + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) # And a creator that does not support copying - if not is_macos_brew(PythonInfo.from_exe(python)): + if not is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): original_init = api.ViaGlobalRefMeta.__init__ def new_init(self, *args, **kwargs): @@ -751,7 +795,7 @@ def new_init(self, *args, **kwargs): # Then a RuntimeError should be raised with a detailed message assert "neither symlink or copy method supported" in str(excinfo.value) assert "symlink: the filesystem does not supports symlink" in str(excinfo.value) - if is_macos_brew(PythonInfo.from_exe(python)): + if is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): assert "copy: Brew disables copy creation" in str(excinfo.value) else: assert "copy: copying is not supported" in str(excinfo.value) diff --git a/tests/unit/create/test_interpreters.py b/tests/unit/create/test_interpreters.py index ae4452b13..b842891f1 100644 --- a/tests/unit/create/test_interpreters.py +++ b/tests/unit/create/test_interpreters.py @@ -5,6 +5,7 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run @@ -18,15 +19,14 @@ def test_failed_to_find_bad_spec(): assert repr(context.value) == msg -SYSTEM = PythonInfo.current_system() - - -@pytest.mark.parametrize( - "of_id", - ({sys.executable} if sys.executable != SYSTEM.executable else set()) | {SYSTEM.implementation}, -) -def test_failed_to_find_implementation(of_id, mocker): - mocker.patch("virtualenv.run.plugin.creators.CreatorSelector._OPTIONS", return_value={}) - with pytest.raises(RuntimeError) as context: - cli_run(["-p", of_id]) - assert repr(context.value) == repr(RuntimeError(f"No virtualenv implementation for {PythonInfo.current_system()}")) +def test_failed_to_find_implementation(mocker, session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + system = PythonInfo.current_system(session_app_data, cache) + of_ids = ({sys.executable} if sys.executable != system.executable else set()) | {system.implementation} + for of_id in of_ids: + mocker.patch("virtualenv.run.plugin.creators.CreatorSelector._OPTIONS", return_value={}) + with pytest.raises(RuntimeError) as context: + cli_run(["-p", of_id]) + assert repr(context.value) == repr( + RuntimeError(f"No virtualenv implementation for {PythonInfo.current_system(session_app_data, cache)}"), + ) diff --git a/tests/unit/create/via_global_ref/test_build_c_ext.py b/tests/unit/create/via_global_ref/test_build_c_ext.py index 1e3ecc069..b00a3eb47 100644 --- a/tests/unit/create/via_global_ref/test_build_c_ext.py +++ b/tests/unit/create/via_global_ref/test_build_c_ext.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import shutil import subprocess @@ -8,23 +9,12 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run from virtualenv.run.plugin.creators import CreatorSelector -CURRENT = PythonInfo.current_system() -CREATOR_CLASSES = CreatorSelector.for_interpreter(CURRENT).key_to_class - - -def builtin_shows_marker_missing(): - builtin_classs = CREATOR_CLASSES.get("builtin") - if builtin_classs is None: - return False - host_include_marker = getattr(builtin_classs, "host_include_marker", None) - if host_include_marker is None: - return False - marker = host_include_marker(CURRENT) - return not marker.exists() +logger = logging.getLogger(__name__) @pytest.mark.slow @@ -33,44 +23,63 @@ def builtin_shows_marker_missing(): strict=False, reason="did not manage to setup CI to run with VC 14.1 C++ compiler, but passes locally", ) -@pytest.mark.skipif( - not Path(CURRENT.system_include).exists() and not builtin_shows_marker_missing(), - reason="Building C-Extensions requires header files with host python", -) -@pytest.mark.parametrize("creator", [i for i in CREATOR_CLASSES if i != "builtin"]) -def test_can_build_c_extensions(creator, tmp_path, coverage_env): - env, greet = tmp_path / "env", str(tmp_path / "greet") - shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) - session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) - coverage_env() - setuptools_index_args = () - if CURRENT.version_info >= (3, 12): - # requires to be able to install setuptools as build dependency - setuptools_index_args = ( - "--find-links", - "https://pypi.org/simple/setuptools/", - ) +def test_can_build_c_extensions(tmp_path, coverage_env, session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + current = PythonInfo.current_system(session_app_data, cache) + creator_classes = CreatorSelector.for_interpreter(current).key_to_class + + logger.warning("system_include: %s", current.system_include) + logger.warning("system_include exists: %s", Path(current.system_include).exists()) - cmd = [ - str(session.creator.script("pip")), - "install", - "--no-index", - *setuptools_index_args, - "--no-deps", - "--disable-pip-version-check", - "-vvv", - greet, - ] - process = Popen(cmd) - process.communicate() - assert process.returncode == 0 + def builtin_shows_marker_missing(): + builtin_classs = creator_classes.get("builtin") + if builtin_classs is None: + return False + host_include_marker = getattr(builtin_classs, "host_include_marker", None) + if host_include_marker is None: + return False + marker = host_include_marker(current) + logger.warning("builtin marker: %s", marker) + logger.warning("builtin marker exists: %s", marker.exists()) + return not marker.exists() - process = Popen( - [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], - universal_newlines=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) - out, _ = process.communicate() - assert process.returncode == 0 - assert out == "Hello World!\n" + system_include = current.system_include + if not Path(system_include).exists() and not builtin_shows_marker_missing(): + pytest.skip("Building C-Extensions requires header files with host python") + + for creator in [i for i in creator_classes if i != "builtin"]: + env, greet = tmp_path / creator / "env", str(tmp_path / creator / "greet") + shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) + session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) + coverage_env() + setuptools_index_args = () + if current.version_info >= (3, 12): + # requires to be able to install setuptools as build dependency + setuptools_index_args = ( + "--find-links", + "https://pypi.org/simple/setuptools/", + ) + + cmd = [ + str(session.creator.script("pip")), + "install", + "--no-index", + *setuptools_index_args, + "--no-deps", + "--disable-pip-version-check", + "-vvv", + greet, + ] + process = Popen(cmd) + process.communicate() + assert process.returncode == 0 + + process = Popen( + [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], + universal_newlines=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ) + out, _ = process.communicate() + assert process.returncode == 0 + assert out == "Hello World!\n" diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index 0e231e022..fed641a98 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -14,13 +14,14 @@ import pytest +from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew from virtualenv.discovery import cached_py_info from virtualenv.discovery.py_info import PythonInfo, VersionInfo from virtualenv.discovery.py_spec import PythonSpec from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink -CURRENT = PythonInfo.current_system() +CURRENT = PythonInfo.current_system(MockAppData(), MockCache()) def test_current_as_json(): @@ -32,19 +33,21 @@ def test_current_as_json(): assert parsed["free_threaded"] is f -def test_bad_exe_py_info_raise(tmp_path, session_app_data): +def test_bad_exe_py_info_raise(tmp_path): exe = str(tmp_path) + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError) as context: - PythonInfo.from_exe(exe, session_app_data) + PythonInfo.from_exe(exe, app_data, cache) msg = str(context.value) assert "code" in msg assert exe in msg -def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys, session_app_data): +def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys): caplog.set_level(logging.NOTSET) exe = str(tmp_path) - result = PythonInfo.from_exe(exe, session_app_data, raise_on_error=False) + app_data, cache = MockAppData(), MockCache() + result = PythonInfo.from_exe(exe, app_data, cache, raise_on_error=False) assert result is None out, _ = capsys.readouterr() assert not out @@ -123,41 +126,45 @@ def test_satisfy_not_version(spec): assert matches is False -def test_py_info_cached_error(mocker, tmp_path, session_app_data): +def test_py_info_cached_error(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) + PythonInfo.from_exe(str(tmp_path), app_data, cache) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) + PythonInfo.from_exe(str(tmp_path), app_data, cache) assert spy.call_count == 1 @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink_error(mocker, tmp_path, session_app_data): +def test_py_info_cached_symlink_error(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) + PythonInfo.from_exe(str(tmp_path), app_data, cache) symlinked = tmp_path / "a" symlinked.symlink_to(tmp_path) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(symlinked), session_app_data) + PythonInfo.from_exe(str(symlinked), app_data, cache) assert spy.call_count == 2 -def test_py_info_cache_clear(mocker, session_app_data): +def test_py_info_cache_clear(mocker): spy = mocker.spy(cached_py_info, "_run_subprocess") - result = PythonInfo.from_exe(sys.executable, session_app_data) + app_data, cache = MockAppData(), MockCache() + result = PythonInfo.from_exe(sys.executable, app_data, cache) assert result is not None count = 1 if result.executable == sys.executable else 2 # at least two, one for the venv, one more for the host assert spy.call_count >= count - PythonInfo.clear_cache(session_app_data) - assert PythonInfo.from_exe(sys.executable, session_app_data) is not None + PythonInfo.clear_cache() + assert PythonInfo.from_exe(sys.executable, app_data, cache) is not None assert spy.call_count >= 2 * count -def test_py_info_cache_invalidation_on_py_info_change(mocker, session_app_data): +def test_py_info_cache_invalidation_on_py_info_change(mocker): # 1. Get a PythonInfo object for the current executable, this will cache it. - PythonInfo.from_exe(sys.executable, session_app_data) + app_data, cache = MockAppData(), MockCache() + PythonInfo.from_exe(sys.executable, app_data, cache) # 2. Spy on _run_subprocess spy = mocker.spy(cached_py_info, "_run_subprocess") @@ -175,7 +182,7 @@ def test_py_info_cache_invalidation_on_py_info_change(mocker, session_app_data): py_info_script.write_text(original_content + "\n# a comment", encoding="utf-8") # 6. Get the PythonInfo object again - info = PythonInfo.from_exe(sys.executable, session_app_data) + info = PythonInfo.from_exe(sys.executable, app_data, cache) # 7. Assert that _run_subprocess was called again native_difference = 1 if info.system_executable == info.executable else 0 @@ -197,9 +204,10 @@ def test_py_info_cache_invalidation_on_py_info_change(mocker, session_app_data): reason="symlink is not supported", ) @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): +def test_py_info_cached_symlink(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") - first_result = PythonInfo.from_exe(sys.executable, session_app_data) + app_data, cache = MockAppData(), MockCache() + first_result = PythonInfo.from_exe(sys.executable, app_data, cache) assert first_result is not None count = spy.call_count # at least two, one for the venv, one more for the host @@ -212,7 +220,7 @@ def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): if pyvenv.exists(): (tmp_path / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") new_exe_str = str(new_exe) - second_result = PythonInfo.from_exe(new_exe_str, session_app_data) + second_result = PythonInfo.from_exe(new_exe_str, app_data, cache) assert second_result.executable == new_exe_str assert spy.call_count == count + 1 # no longer needed the host invocation, but the new symlink is must @@ -259,10 +267,10 @@ def test_system_executable_no_exact_match( # noqa: PLR0913 tmp_path, mocker, caplog, - session_app_data, ): """Here we should fallback to other compatible""" caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() def _make_py_info(of): base = copy.deepcopy(CURRENT) @@ -290,15 +298,15 @@ def _make_py_info(of): mocker.patch.object(target_py_info, "_find_possible_exe_names", return_value=names) mocker.patch.object(target_py_info, "_find_possible_folders", return_value=[str(tmp_path)]) - def func(k, app_data, resolve_to_host, raise_on_error, env, cache=None): # noqa: ARG001, PLR0913 - return discovered_with_path[k] + def func(exe, app_data, cache, raise_on_error=True, ignore_cache=False, resolve_to_host=True, env=None): # noqa: ARG001, PLR0913 + return discovered_with_path.get(exe) - mocker.patch.object(target_py_info, "from_exe", side_effect=func) + mocker.patch.object(PythonInfo, "from_exe", side_effect=func) target_py_info.real_prefix = str(tmp_path) target_py_info.system_executable = None target_py_info.executable = str(tmp_path) - mapped = target_py_info._resolve_to_system(session_app_data, target_py_info) # noqa: SLF001 + mapped = target_py_info._resolve_to_system(app_data, target_py_info, cache) # noqa: SLF001 assert mapped.system_executable == CURRENT.system_executable found = discovered_with_path[mapped.base_executable] assert found is selected @@ -325,7 +333,8 @@ def test_py_info_ignores_distutils_config(monkeypatch, tmp_path): """ (tmp_path / "setup.cfg").write_text(dedent(raw), encoding="utf-8") monkeypatch.chdir(tmp_path) - py_info = PythonInfo.from_exe(sys.executable) + app_data, cache = MockAppData(), MockCache() + py_info = PythonInfo.from_exe(sys.executable, app_data, cache) distutils = py_info.distutils_install for key, value in distutils.items(): assert not value.startswith(str(tmp_path)), f"{key}={value}" @@ -362,10 +371,11 @@ def test_py_info_setuptools(): @pytest.mark.usefixtures("_skip_if_test_in_system") -def test_py_info_to_system_raises(session_app_data, mocker, caplog): +def test_py_info_to_system_raises(mocker, caplog): caplog.set_level(logging.DEBUG) mocker.patch.object(PythonInfo, "_find_possible_folders", return_value=[]) - result = PythonInfo.from_exe(sys.executable, app_data=session_app_data, raise_on_error=False) + app_data, cache = MockAppData(), MockCache() + result = PythonInfo.from_exe(sys.executable, app_data=app_data, cache=cache, raise_on_error=False) assert result is None log = caplog.records[-1] assert log.levelno == logging.INFO diff --git a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py index 90894a59c..f3846422e 100644 --- a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py +++ b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py @@ -6,15 +6,17 @@ import pytest +from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.py_info import EXTENSIONS, PythonInfo from virtualenv.info import IS_WIN, fs_is_case_sensitive, fs_supports_symlink -CURRENT = PythonInfo.current() +CURRENT = PythonInfo.current(MockAppData(), MockCache()) -def test_discover_empty_folder(tmp_path, session_app_data): +def test_discover_empty_folder(tmp_path): + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - CURRENT.discover_exe(session_app_data, prefix=str(tmp_path)) + CURRENT.discover_exe(app_data, cache, prefix=str(tmp_path)) BASE = (CURRENT.install_path("scripts"), ".") @@ -26,8 +28,9 @@ def test_discover_empty_folder(tmp_path, session_app_data): @pytest.mark.parametrize("arch", [CURRENT.architecture, ""]) @pytest.mark.parametrize("version", [".".join(str(i) for i in CURRENT.version_info[0:i]) for i in range(3, 0, -1)]) @pytest.mark.parametrize("impl", [CURRENT.implementation, "python"]) -def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, session_app_data): # noqa: PLR0913 +def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog): # noqa: PLR0913 caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() folder = tmp_path / into folder.mkdir(parents=True, exist_ok=True) name = f"{impl}{version}{'t' if CURRENT.free_threaded else ''}" @@ -40,7 +43,7 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, sessio if pyvenv.exists(): (folder / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") inside_folder = str(tmp_path) - base = CURRENT.discover_exe(session_app_data, inside_folder) + base = CURRENT.discover_exe(app_data, cache, inside_folder) found = base.executable dest_str = str(dest) if not fs_is_case_sensitive(): @@ -53,4 +56,4 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, sessio dest.rename(dest.parent / (dest.name + "-1")) CURRENT._cache_exe_discovery.clear() # noqa: SLF001 with pytest.raises(RuntimeError): - CURRENT.discover_exe(session_app_data, inside_folder) + CURRENT.discover_exe(app_data, cache, inside_folder) diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index e6c78e2e3..f517101e2 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -11,6 +11,7 @@ import pytest +from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.builtin import Builtin, LazyPathDump, get_interpreter, get_paths from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_WIN, fs_supports_symlink @@ -19,9 +20,10 @@ @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") @pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) @pytest.mark.parametrize("specificity", ["more", "less", "none"]) -def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog, session_app_data): # noqa: PLR0913 +def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog): caplog.set_level(logging.DEBUG) - current = PythonInfo.current_system(session_app_data) + app_data, cache = MockAppData(), MockCache() + current = PythonInfo.current_system(app_data, cache) name = "somethingVeryCryptic" threaded = "t" if current.free_threaded else "" if case == "lower": @@ -51,36 +53,39 @@ def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog, se (target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes()) new_path = os.pathsep.join([str(target), *os.environ.get("PATH", "").split(os.pathsep)]) monkeypatch.setenv("PATH", new_path) - interpreter = get_interpreter(core, []) + interpreter = get_interpreter(core, [], app_data, cache) assert interpreter is not None def test_discovery_via_path_not_found(tmp_path, monkeypatch): monkeypatch.setenv("PATH", str(tmp_path)) - interpreter = get_interpreter(uuid4().hex, []) + app_data, cache = MockAppData(), MockCache() + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) assert interpreter is None def test_discovery_via_path_in_nonbrowseable_directory(tmp_path, monkeypatch): bad_perm = tmp_path / "bad_perm" bad_perm.mkdir(mode=0o000) + app_data, cache = MockAppData(), MockCache() # path entry is unreadable monkeypatch.setenv("PATH", str(bad_perm)) - interpreter = get_interpreter(uuid4().hex, []) + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) assert interpreter is None # path entry parent is unreadable monkeypatch.setenv("PATH", str(bad_perm / "bin")) - interpreter = get_interpreter(uuid4().hex, []) + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) assert interpreter is None -def test_relative_path(session_app_data, monkeypatch): - sys_executable = Path(PythonInfo.current_system(app_data=session_app_data).system_executable) +def test_relative_path(monkeypatch): + app_data, cache = MockAppData(), MockCache() + sys_executable = Path(PythonInfo.current_system(app_data=app_data, cache=cache).system_executable) cwd = sys_executable.parents[1] monkeypatch.chdir(str(cwd)) relative = str(sys_executable.relative_to(cwd)) - result = get_interpreter(relative, [], session_app_data) + result = get_interpreter(relative, [], app_data, cache) assert result is not None @@ -95,13 +100,14 @@ def test_uv_python(monkeypatch, tmp_path_factory, mocker): with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: m.setenv("UV_PYTHON_INSTALL_DIR", str(uv_python_install_dir)) - get_interpreter("python", []) + app_data, cache = MockAppData(), MockCache() + get_interpreter("python", [], app_data, cache) mock_from_exe.assert_not_called() bin_path = uv_python_install_dir.joinpath("some-py-impl", "bin") bin_path.mkdir(parents=True) bin_path.joinpath("python").touch() - get_interpreter("python", []) + get_interpreter("python", [], app_data, cache) mock_from_exe.assert_called_once() assert mock_from_exe.call_args[0][0] == str(bin_path / "python") @@ -120,13 +126,14 @@ def test_uv_python(monkeypatch, tmp_path_factory, mocker): with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: m.setenv("XDG_DATA_HOME", str(xdg_data_home)) - get_interpreter("python", []) + app_data, cache = MockAppData(), MockCache() + get_interpreter("python", [], app_data, cache) mock_from_exe.assert_not_called() bin_path = xdg_data_home.joinpath("uv", "python", "some-py-impl", "bin") bin_path.mkdir(parents=True) bin_path.joinpath("python").touch() - get_interpreter("python", []) + get_interpreter("python", [], app_data, cache) mock_from_exe.assert_called_once() assert mock_from_exe.call_args[0][0] == str(bin_path / "python") @@ -135,21 +142,24 @@ def test_uv_python(monkeypatch, tmp_path_factory, mocker): with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: m.setattr("virtualenv.discovery.builtin.user_data_path", lambda x: user_data_path / x) - get_interpreter("python", []) + app_data, cache = MockAppData(), MockCache() + get_interpreter("python", [], app_data, cache) mock_from_exe.assert_not_called() bin_path = user_data_path.joinpath("uv", "python", "some-py-impl", "bin") bin_path.mkdir(parents=True) bin_path.joinpath("python").touch() - get_interpreter("python", []) + get_interpreter("python", [], app_data, cache) mock_from_exe.assert_called_once() assert mock_from_exe.call_args[0][0] == str(bin_path / "python") -def test_discovery_fallback_fail(session_app_data, caplog): +def test_discovery_fallback_fail(caplog): caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ), + Namespace(app_data=app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ), + cache, ) result = builtin.run() @@ -158,10 +168,12 @@ def test_discovery_fallback_fail(session_app_data, caplog): assert "accepted" not in caplog.text -def test_discovery_fallback_ok(session_app_data, caplog): +def test_discovery_fallback_ok(caplog): caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ), + Namespace(app_data=app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ), + cache, ) result = builtin.run() @@ -180,10 +192,12 @@ def mock_get_interpreter(mocker): @pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocker, monkeypatch, session_app_data): +def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocker, monkeypatch): monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") + app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_env_var"], env=os.environ), + Namespace(app_data=app_data, try_first_with=[], python=["python_from_env_var"], env=os.environ), + cache, ) result = builtin.run() @@ -192,17 +206,17 @@ def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocke @pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_second_python_specified_when_more_than_one_is_specified_and_env_var_is_specified( - mocker, monkeypatch, session_app_data -): +def test_returns_second_python_specified_when_more_than_one_is_specified_and_env_var_is_specified(mocker, monkeypatch): monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") + app_data, cache = MockAppData(), MockCache() builtin = Builtin( Namespace( - app_data=session_app_data, + app_data=app_data, try_first_with=[], python=["python_from_env_var", "python_from_cli"], env=os.environ, ), + cache, ) result = builtin.run() @@ -210,7 +224,7 @@ def test_returns_second_python_specified_when_more_than_one_is_specified_and_env assert result == mocker.sentinel.python_from_cli -def test_discovery_absolute_path_with_try_first(tmp_path, session_app_data): +def test_discovery_absolute_path_with_try_first(tmp_path): good_env = tmp_path / "good" bad_env = tmp_path / "bad" @@ -226,10 +240,12 @@ def test_discovery_absolute_path_with_try_first(tmp_path, session_app_data): # The spec is an absolute path, this should be a hard requirement. # The --try-first-with option should be rejected as it does not match the spec. + app_data, cache = MockAppData(), MockCache() interpreter = get_interpreter( str(good_exe), try_first_with=[str(bad_exe)], - app_data=session_app_data, + app_data=app_data, + cache=cache, ) assert interpreter is not None @@ -240,7 +256,8 @@ def test_discovery_via_path_with_file(tmp_path, monkeypatch): a_file = tmp_path / "a_file" a_file.touch() monkeypatch.setenv("PATH", str(a_file)) - interpreter = get_interpreter(uuid4().hex, []) + app_data, cache = MockAppData(), MockCache() + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) assert interpreter is None @@ -320,10 +337,12 @@ def test_lazy_path_dump_debug(monkeypatch, tmp_path): @pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_first_python_specified_when_no_env_var_is_specified(mocker, monkeypatch, session_app_data): +def test_returns_first_python_specified_when_no_env_var_is_specified(mocker, monkeypatch): monkeypatch.delenv("VIRTUALENV_PYTHON", raising=False) + app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_cli"], env=os.environ), + Namespace(app_data=app_data, try_first_with=[], python=["python_from_cli"], env=os.environ), + cache, ) result = builtin.run() diff --git a/tests/unit/discovery/util.py b/tests/unit/discovery/util.py new file mode 100644 index 000000000..7908c7df8 --- /dev/null +++ b/tests/unit/discovery/util.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, ContextManager + +if TYPE_CHECKING: + from pathlib import Path + + +class MockAppData: + def __init__(self, readonly: bool = False) -> None: + self.readonly = readonly + self._py_info_clear_called = 0 + self._py_info_map: dict[Path, Any] = {} + + def py_info(self, path: Path) -> Any: + return self._py_info_map.get(path) + + def py_info_clear(self) -> None: + self._py_info_clear_called += 1 + + @contextmanager + def ensure_extracted(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: # noqa: ARG002 + yield path + + @contextmanager + def extract(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: # noqa: ARG002 + yield path + + def close(self) -> None: + pass + + +class MockCache: + def __init__(self) -> None: + self._cache: dict[Path, Any] = {} + self._clear_called = 0 + + def get(self, path: Path) -> Any: + return self._cache.get(path) + + def set(self, path: Path, data: Any) -> None: + self._cache[path] = data + + def remove(self, path: Path) -> None: + if path in self._cache: + del self._cache[path] + + def clear(self) -> None: + self._clear_called += 1 + self._cache.clear() diff --git a/tests/unit/discovery/windows/test_windows.py b/tests/unit/discovery/windows/test_windows.py index 594a1302f..98b849f57 100644 --- a/tests/unit/discovery/windows/test_windows.py +++ b/tests/unit/discovery/windows/test_windows.py @@ -4,6 +4,7 @@ import pytest +from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.py_spec import PythonSpec @@ -36,5 +37,5 @@ def test_propose_interpreters(string_spec, expected_exe): from virtualenv.discovery.windows import propose_interpreters # noqa: PLC0415 spec = PythonSpec.from_string_spec(string_spec) - interpreter = next(propose_interpreters(spec=spec, cache_dir=None, env=None)) + interpreter = next(propose_interpreters(spec, MockAppData(), MockCache(), env=None)) assert interpreter.executable == expected_exe diff --git a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py index 4076573e1..b37b7cdbb 100644 --- a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py +++ b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py @@ -10,6 +10,7 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery import cached_py_info from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink @@ -25,8 +26,16 @@ @pytest.mark.slow @pytest.mark.parametrize("copies", [False, True] if fs_supports_symlink() else [True]) -def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies, for_py_version): # noqa: PLR0915 - current = PythonInfo.current_system() +def test_seed_link_via_app_data( # noqa: PLR0913, PLR0915 + tmp_path, + coverage_env, + current_fastest, + copies, + for_py_version, + session_app_data, +): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + current = PythonInfo.current_system(session_app_data, cache) bundle_ver = BUNDLE_SUPPORT[current.version_release_str] create_cmd = [ str(tmp_path / "en v"), # space in the name to ensure generated scripts work when path has space From 916375d6e3c6702fcd3e71fbc31394911cf5291d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Thu, 14 Aug 2025 01:19:31 -0400 Subject: [PATCH 02/17] feat: ensure creation of python3.exe and python3 on Windows (#2957) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Emre Şafak <3928300+esafak@users.noreply.github.com> Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- docs/changelog/2774.feature.rst | 1 + .../create/via_global_ref/builtin/cpython/common.py | 5 ++++- .../via_global_ref/builtin/cpython/test_cpython3_win.py | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/2774.feature.rst diff --git a/docs/changelog/2774.feature.rst b/docs/changelog/2774.feature.rst new file mode 100644 index 000000000..7821b80cf --- /dev/null +++ b/docs/changelog/2774.feature.rst @@ -0,0 +1 @@ +Ensure python3.exe and python3 on Windows for Python 3. - by :user:`esafak`. diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py index 7c2a04a32..5cc993bda 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py @@ -38,7 +38,10 @@ def _executables(cls, interpreter): # - https://bugs.python.org/issue42013 # - venv host = cls.host_python(interpreter) - for path in (host.parent / n for n in {"python.exe", host.name}): + names = {"python.exe", host.name} + if interpreter.version_info.major == 3: # noqa: PLR2004 + names.update({"python3.exe", "python3"}) + for path in (host.parent / n for n in names): yield host, [path.name], RefMust.COPY, RefWhen.ANY # for more info on pythonw.exe see https://stackoverflow.com/a/30313091 python_w = host.parent / "pythonw.exe" diff --git a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py index f831de114..4367ebc50 100644 --- a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py +++ b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py @@ -99,3 +99,11 @@ def test_no_python_zip_if_not_exists(py_info, mock_files): sources = tuple(CPython3Windows.sources(interpreter=py_info)) assert python_zip in py_info.path assert not contains_ref(sources, python_zip) + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) +def test_python3_exe_present(py_info, mock_files): + mock_files(CPYTHON3_PATH, [py_info.system_executable]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert contains_exe(sources, py_info.system_executable, "python3.exe") + assert contains_exe(sources, py_info.system_executable, "python3") From dad9369e97f5aef7e33777b18dcdb51b1fdac7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:50:46 -0400 Subject: [PATCH 03/17] fix: Use getattr for tcl/tk library paths (#2945) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/changelog/2944.bugfix.rst | 1 + src/virtualenv/activation/bash/__init__.py | 4 ++-- src/virtualenv/activation/fish/__init__.py | 4 ++-- src/virtualenv/activation/nushell/__init__.py | 4 ++-- src/virtualenv/activation/via_template.py | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/2944.bugfix.rst diff --git a/docs/changelog/2944.bugfix.rst b/docs/changelog/2944.bugfix.rst new file mode 100644 index 000000000..21ba0d22e --- /dev/null +++ b/docs/changelog/2944.bugfix.rst @@ -0,0 +1 @@ +Replaced direct references to tcl/tk library paths with getattr. By :user:`esafak` diff --git a/src/virtualenv/activation/bash/__init__.py b/src/virtualenv/activation/bash/__init__.py index fd86a4208..4f160744f 100644 --- a/src/virtualenv/activation/bash/__init__.py +++ b/src/virtualenv/activation/bash/__init__.py @@ -15,8 +15,8 @@ def as_name(self, template): def replacements(self, creator, dest): data = super().replacements(creator, dest) data.update({ - "__TCL_LIBRARY__": creator.interpreter.tcl_lib or "", - "__TK_LIBRARY__": creator.interpreter.tk_lib or "", + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", }) return data diff --git a/src/virtualenv/activation/fish/__init__.py b/src/virtualenv/activation/fish/__init__.py index 28052e64f..26263566e 100644 --- a/src/virtualenv/activation/fish/__init__.py +++ b/src/virtualenv/activation/fish/__init__.py @@ -10,8 +10,8 @@ def templates(self): def replacements(self, creator, dest): data = super().replacements(creator, dest) data.update({ - "__TCL_LIBRARY__": creator.interpreter.tcl_lib or "", - "__TK_LIBRARY__": creator.interpreter.tk_lib or "", + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", }) return data diff --git a/src/virtualenv/activation/nushell/__init__.py b/src/virtualenv/activation/nushell/__init__.py index 9558a70a5..d3b312497 100644 --- a/src/virtualenv/activation/nushell/__init__.py +++ b/src/virtualenv/activation/nushell/__init__.py @@ -34,8 +34,8 @@ def replacements(self, creator, dest_folder): # noqa: ARG002 "__VIRTUAL_ENV__": str(creator.dest), "__VIRTUAL_NAME__": creator.env_name, "__BIN_NAME__": str(creator.bin_dir.relative_to(creator.dest)), - "__TCL_LIBRARY__": creator.interpreter.tcl_lib or "", - "__TK_LIBRARY__": creator.interpreter.tk_lib or "", + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", } diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 83229441a..85f932605 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -47,8 +47,8 @@ def replacements(self, creator, dest_folder): # noqa: ARG002 "__VIRTUAL_NAME__": creator.env_name, "__BIN_NAME__": str(creator.bin_dir.relative_to(creator.dest)), "__PATH_SEP__": os.pathsep, - "__TCL_LIBRARY__": creator.interpreter.tcl_lib or "", - "__TK_LIBRARY__": creator.interpreter.tk_lib or "", + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", } def _generate(self, replacements, templates, to_folder, creator): From 66d793edd347bc8792f5fe16f950c77f23f6b60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 09:37:16 -0400 Subject: [PATCH 04/17] fix: Import fs_is_case_sensitive absolutely in py_info.py (#2960) --- docs/changelog/2774.feature.rst | 2 +- docs/changelog/2944.bugfix.rst | 2 +- docs/changelog/2955.bugfix.rst | 1 + src/virtualenv/discovery/py_info.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/2955.bugfix.rst diff --git a/docs/changelog/2774.feature.rst b/docs/changelog/2774.feature.rst index 7821b80cf..3800244ac 100644 --- a/docs/changelog/2774.feature.rst +++ b/docs/changelog/2774.feature.rst @@ -1 +1 @@ -Ensure python3.exe and python3 on Windows for Python 3. - by :user:`esafak`. +Ensure python3.exe and python3 on Windows for Python 3 - by :user:`esafak`. diff --git a/docs/changelog/2944.bugfix.rst b/docs/changelog/2944.bugfix.rst index 21ba0d22e..6cd6408d9 100644 --- a/docs/changelog/2944.bugfix.rst +++ b/docs/changelog/2944.bugfix.rst @@ -1 +1 @@ -Replaced direct references to tcl/tk library paths with getattr. By :user:`esafak` +Replaced direct references to tcl/tk library paths with getattr - by :user:`esafak` diff --git a/docs/changelog/2955.bugfix.rst b/docs/changelog/2955.bugfix.rst new file mode 100644 index 000000000..b5b935a49 --- /dev/null +++ b/docs/changelog/2955.bugfix.rst @@ -0,0 +1 @@ +Restore absolute import of fs_is_case_sensitive - by :user:`esafak`. diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 797f88bdb..8ae456966 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -660,7 +660,7 @@ def _possible_base(self): lower = base.lower() yield lower - from .info import fs_is_case_sensitive # noqa: PLC0415 + from virtualenv.discovery.info import fs_is_case_sensitive # noqa: PLC0415 if fs_is_case_sensitive(): if base != lower: From 5c5bf430b3e893c80a1bf22933c2c3a800a1b459 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 18:45:46 +0000 Subject: [PATCH 05/17] [pre-commit.ci] pre-commit autoupdate (#2961) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17611776d..6ffe9a65b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.2 + rev: 0.33.3 hooks: - id: check-github-workflows args: ["--verbose"] @@ -24,7 +24,7 @@ repos: hooks: - id: pyproject-fmt - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.12.8" + rev: "v0.12.9" hooks: - id: ruff-format - id: ruff From 9081de13c3b5cf45a747b0ea1793ba4622c200b5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 07:42:57 -0700 Subject: [PATCH 06/17] [pre-commit.ci] pre-commit autoupdate (#2962) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ffe9a65b..f39fcec3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: pyproject-fmt - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.12.9" + rev: "v0.12.11" hooks: - id: ruff-format - id: ruff From 6e1723a861171d6a4e5c7e45c1fb49abc8aec581 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 09:03:12 -0700 Subject: [PATCH 07/17] chore(deps): bump pypa/gh-action-pypi-publish from 1.12.3 to 1.13.0 in /.github/workflows (#2964) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 820c63f65..f581f3e54 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -44,6 +44,6 @@ jobs: name: ${{ env.dists-artifact-name }} path: dist/ - name: 🚀 Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.12.3 + uses: pypa/gh-action-pypi-publish@v1.13.0 with: attestations: true From 81d5187bc92056d204d297935cfb006a4986dfb0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Sep 2025 16:17:40 -0700 Subject: [PATCH 08/17] [pre-commit.ci] pre-commit autoupdate (#2965) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f39fcec3b..cbe29d7df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: pyproject-fmt - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.12.11" + rev: "v0.13.0" hooks: - id: ruff-format - id: ruff From 89dc16bfe3a65b32b69bfaa7a882902f7a53c132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Wed, 8 Oct 2025 13:25:54 -0700 Subject: [PATCH 09/17] Declare 3.14 support (#2970) --- .github/workflows/check.yaml | 12 +++++++----- .github/workflows/release.yaml | 2 +- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 3 ++- src/virtualenv/util/path/_sync.py | 2 +- tox.ini | 5 +++++ 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 3f943853f..68120bde6 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -21,6 +21,8 @@ jobs: fail-fast: false matrix: py: + - "3.14t" + - "3.14" - "3.13t" - "3.13" - "3.12" @@ -56,14 +58,14 @@ jobs: - name: 🐍 Setup Python for tox uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" - name: 📦 Install tox with this virtualenv shell: bash run: | - if [[ "${{ matrix.py }}" == "3.13t" ]]; then - uv tool install --no-managed-python --python 3.13 tox --with . + if [[ "${{ matrix.py }}" == "3.13t" || "${{ matrix.py }}" == "3.14t" ]]; then + uv tool install --no-managed-python --python 3.14 tox --with . else - uv tool install --no-managed-python --python 3.13 tox --with tox-uv --with . + uv tool install --no-managed-python --python 3.14 tox --with tox-uv --with . fi - name: 🐍 Setup Python for test ${{ matrix.py }} uses: actions/setup-python@v5 @@ -135,7 +137,7 @@ jobs: - name: 🚀 Install uv uses: astral-sh/setup-uv@v4 - name: 📦 Install tox - run: uv tool install --python-preference only-managed --python 3.13 tox --with tox-uv + run: uv tool install --python-preference only-managed --python 3.14 tox --with tox-uv - name: 📥 Checkout code uses: actions/checkout@v4 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f581f3e54..def30f5d9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -21,7 +21,7 @@ jobs: cache-dependency-glob: "pyproject.toml" github-token: ${{ secrets.GITHUB_TOKEN }} - name: 📦 Build package - run: uv build --python 3.13 --python-preference only-managed --sdist --wheel . --out-dir dist + run: uv build --python 3.14 --python-preference only-managed --sdist --wheel . --out-dir dist - name: 📦 Store the distribution packages uses: actions/upload-artifact@v4 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cbe29d7df..332ac436b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.3 + rev: 0.34.0 hooks: - id: check-github-workflows args: ["--verbose"] @@ -20,11 +20,11 @@ repos: - id: tox-ini-fmt args: ["-p", "fix"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "v2.6.0" + rev: "v2.8.0" hooks: - id: pyproject-fmt - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.13.0" + rev: "v0.14.0" hooks: - id: ruff-format - id: ruff diff --git a/pyproject.toml b/pyproject.toml index f879b4034..85ff87ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries", @@ -159,7 +160,7 @@ builtin = "clear,usage,en-GB_to_en-US" count = true [tool.pyproject-fmt] -max_supported_python = "3.13" +max_supported_python = "3.14" [tool.pytest.ini_options] markers = [ diff --git a/src/virtualenv/util/path/_sync.py b/src/virtualenv/util/path/_sync.py index 02a6f6e9e..4dd5a9860 100644 --- a/src/virtualenv/util/path/_sync.py +++ b/src/virtualenv/util/path/_sync.py @@ -11,7 +11,7 @@ def ensure_dir(path): if not path.exists(): - LOGGER.debug("create folder %s", str(path)) + LOGGER.debug("create folder %s", path) os.makedirs(str(path)) diff --git a/tox.ini b/tox.ini index d8aefed7c..7cdebdbe7 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ requires = env_list = fix pypy3 + 3.14 3.13 3.12 3.11 @@ -14,6 +15,7 @@ env_list = coverage readme docs + 3.14t 3.13t skip_missing_interpreters = true @@ -70,6 +72,9 @@ commands = sphinx-build -d "{envtmpdir}/doctree" docs "{toxworkdir}/docs_out" --color -b html {posargs:-W} python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' +[testenv:3.14t] +base_python = {env:TOX_BASEPYTHON} + [testenv:3.13t] base_python = {env:TOX_BASEPYTHON} From 8e1ecc705a502705931e7576b8dd72bab90cde91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Wed, 8 Oct 2025 13:27:05 -0700 Subject: [PATCH 10/17] release 20.35.0 (#2971) --- docs/changelog.rst | 13 +++++++++++++ docs/changelog/2074.feature.rst | 1 - docs/changelog/2774.feature.rst | 1 - docs/changelog/2944.bugfix.rst | 1 - docs/changelog/2955.bugfix.rst | 1 - 5 files changed, 13 insertions(+), 4 deletions(-) delete mode 100644 docs/changelog/2074.feature.rst delete mode 100644 docs/changelog/2774.feature.rst delete mode 100644 docs/changelog/2944.bugfix.rst delete mode 100644 docs/changelog/2955.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index e333f091f..c37eda600 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,19 @@ Release History .. towncrier release notes start +v20.35.0 (2025-10-08) +--------------------- + +Features - 20.35.0 +~~~~~~~~~~~~~~~~~~ +- Add AppData and Cache protocols to discovery for decoupling - by :user:`esafak`. (:issue:`2074`) +- Ensure python3.exe and python3 on Windows for Python 3 - by :user:`esafak`. (:issue:`2774`) + +Bugfixes - 20.35.0 +~~~~~~~~~~~~~~~~~~ +- Replaced direct references to tcl/tk library paths with getattr - by :user:`esafak` (:issue:`2944`) +- Restore absolute import of fs_is_case_sensitive - by :user:`esafak`. (:issue:`2955`) + v20.34.0 (2025-08-13) --------------------- diff --git a/docs/changelog/2074.feature.rst b/docs/changelog/2074.feature.rst deleted file mode 100644 index 61b1c38b5..000000000 --- a/docs/changelog/2074.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add AppData and Cache protocols to discovery for decoupling - by :user:`esafak`. diff --git a/docs/changelog/2774.feature.rst b/docs/changelog/2774.feature.rst deleted file mode 100644 index 3800244ac..000000000 --- a/docs/changelog/2774.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure python3.exe and python3 on Windows for Python 3 - by :user:`esafak`. diff --git a/docs/changelog/2944.bugfix.rst b/docs/changelog/2944.bugfix.rst deleted file mode 100644 index 6cd6408d9..000000000 --- a/docs/changelog/2944.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Replaced direct references to tcl/tk library paths with getattr - by :user:`esafak` diff --git a/docs/changelog/2955.bugfix.rst b/docs/changelog/2955.bugfix.rst deleted file mode 100644 index b5b935a49..000000000 --- a/docs/changelog/2955.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Restore absolute import of fs_is_case_sensitive - by :user:`esafak`. From de1820dd472786383f0ee393524141880fe4e055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:51:57 -0400 Subject: [PATCH 11/17] fix: Patch get_interpreter to handle missing cache and app_data (#2974) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/changelog/2972.bugfix.rst | 1 + src/virtualenv/discovery/builtin.py | 19 +++++++++++++++++++ tests/unit/discovery/test_discovery.py | 12 ++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 docs/changelog/2972.bugfix.rst diff --git a/docs/changelog/2972.bugfix.rst b/docs/changelog/2972.bugfix.rst new file mode 100644 index 000000000..5e38f237e --- /dev/null +++ b/docs/changelog/2972.bugfix.rst @@ -0,0 +1 @@ +Patch get_interpreter to handle missing cache and app_data - by :user:`esafak` diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index 3c6dcb2fd..1e4364db5 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -77,6 +77,25 @@ def get_interpreter( cache=None, env: Mapping[str, str] | None = None, ) -> PythonInfo | None: + """ + Find an interpreter that matches a given specification. + + :param key: the specification of the interpreter to find + :param try_first_with: a list of interpreters to try first + :param app_data: the application data folder + :param cache: a cache of python information + :param env: the environment to use + :return: the interpreter if found, otherwise None + """ + if cache is None: + # Import locally to avoid a circular dependency + from virtualenv.app_data import AppDataDisabled # noqa: PLC0415 + from virtualenv.cache import FileCache # noqa: PLC0415 + + if app_data is None: + app_data = AppDataDisabled() + cache = FileCache(store_factory=app_data.py_info, clearer=app_data.py_info_clear) + spec = PythonSpec.from_string_spec(key) LOGGER.info("find interpreter for spec %r", spec) proposed_paths = set() diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index f517101e2..68de1da56 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -224,6 +224,18 @@ def test_returns_second_python_specified_when_more_than_one_is_specified_and_env assert result == mocker.sentinel.python_from_cli +def test_get_interpreter_no_cache_no_app_data(): + """Test that get_interpreter can be called without cache and app_data.""" + # A call to a valid interpreter should succeed and return a PythonInfo object. + interpreter = get_interpreter(sys.executable, []) + assert interpreter is not None + assert Path(interpreter.executable).is_file() + + # A call to an invalid interpreter should not fail and should return None. + interpreter = get_interpreter("a-python-that-does-not-exist", []) + assert interpreter is None + + def test_discovery_absolute_path_with_try_first(tmp_path): good_env = tmp_path / "good" bad_env = tmp_path / "bad" From 2dbf4f2269b87782ddf9a9a0e6ff47096f9f62cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Thu, 9 Oct 2025 15:20:19 -0700 Subject: [PATCH 12/17] Fix backwards incompatible changes on PythonInfo (#2975) --- .github/release.yml | 4 +- docs/changelog/2975.bugfix.rst | 1 + src/virtualenv/discovery/py_info.py | 39 ++++++++----------- tests/unit/create/conftest.py | 2 +- .../py_info/test_py_info_exe_based_of.py | 6 +-- 5 files changed, 24 insertions(+), 28 deletions(-) create mode 100644 docs/changelog/2975.bugfix.rst diff --git a/.github/release.yml b/.github/release.yml index 9d1e0987b..5f89818f9 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -1,5 +1,5 @@ changelog: exclude: authors: - - dependabot - - pre-commit-ci + - dependabot[bot] + - pre-commit-ci[bot] diff --git a/docs/changelog/2975.bugfix.rst b/docs/changelog/2975.bugfix.rst new file mode 100644 index 000000000..87f3ba69a --- /dev/null +++ b/docs/changelog/2975.bugfix.rst @@ -0,0 +1 @@ +Fix backwards incompatible changes to ``PythonInfo`` - by :user:`gaborbernat`. diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 8ae456966..960421c30 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -27,10 +27,10 @@ def _get_path_extensions(): EXTENSIONS = _get_path_extensions() -_CONF_VAR_RE = re.compile(r"\{\w+\}") +_CONF_VAR_RE = re.compile(r"\{\w+}") -class PythonInfo: +class PythonInfo: # noqa: PLR0904 """Contains information for a Python interpreter.""" def __init__(self) -> None: # noqa: PLR0915 @@ -135,6 +135,7 @@ def abs_path(v): self.system_stdlib = self.sysconfig_path("stdlib", confs) self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs) self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) + self._creators = None @staticmethod def _get_tcl_tk_libs(): @@ -310,6 +311,13 @@ def sysconfig_path(self, key, config_var=None, sep=os.sep): config_var = base return pattern.format(**config_var).replace("/", sep) + def creators(self, refresh=False): # noqa: FBT002 + if self._creators is None or refresh is True: + from virtualenv.run.plugin.creators import CreatorSelector # noqa: PLC0415 + + self._creators = CreatorSelector.for_interpreter(self) + return self._creators + @property def system_include(self): path = self.sysconfig_path( @@ -423,7 +431,7 @@ def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 _current = None @classmethod - def current(cls, app_data, cache): + def current(cls, app_data=None, cache=None): """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. @@ -439,7 +447,7 @@ def current(cls, app_data, cache): return cls._current @classmethod - def current_system(cls, app_data, cache) -> PythonInfo: + def current_system(cls, app_data=None, cache=None) -> PythonInfo: """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. @@ -467,8 +475,8 @@ def _to_dict(self): def from_exe( # noqa: PLR0913 cls, exe, - app_data, - cache, + app_data=None, + cache=None, raise_on_error=True, # noqa: FBT002 ignore_cache=False, # noqa: FBT002 resolve_to_host=True, # noqa: FBT002 @@ -480,13 +488,7 @@ def from_exe( # noqa: PLR0913 env = os.environ if env is None else env proposed = from_exe_cache( - cls, - app_data, - exe, - env=env, - raise_on_error=raise_on_error, - ignore_cache=ignore_cache, - cache=cache, + cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache, cache=cache ) if isinstance(proposed, PythonInfo) and resolve_to_host: @@ -538,7 +540,7 @@ def _resolve_to_system(cls, app_data, target, cache): _cache_exe_discovery = {} # noqa: RUF012 - def discover_exe(self, app_data, cache, prefix, exact=True, env=None): # noqa: FBT002 + def discover_exe(self, app_data, prefix, exact=True, env=None, cache=None): # noqa: FBT002 key = prefix, exact if key in self._cache_exe_discovery and prefix: LOGGER.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) @@ -568,14 +570,7 @@ def _check_exe(self, app_data, cache, folder, name, exact, discovered, env): # exe_path = os.path.join(folder, name) if not os.path.exists(exe_path): return None - info = self.from_exe( - exe_path, - app_data, - cache, - resolve_to_host=False, - raise_on_error=False, - env=env, - ) + info = self.from_exe(exe_path, app_data, cache, resolve_to_host=False, raise_on_error=False, env=env) if info is None: # ignore if for some reason we can't query return None for item in ["implementation", "architecture", "version_info"]: diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py index 4c7bf45c0..bb897dd12 100644 --- a/tests/unit/create/conftest.py +++ b/tests/unit/create/conftest.py @@ -38,7 +38,7 @@ def venv(tmp_path_factory, session_app_data, current_info): # sadly creating a virtual environment does not tell us where the executable lives in general case # so discover using some heuristic cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - return current_info.discover_exe(session_app_data, cache, prefix=str(dest)).original_executable + return current_info.discover_exe(session_app_data, prefix=str(dest), cache=cache).original_executable PYTHON = { diff --git a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py index f3846422e..ec3def025 100644 --- a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py +++ b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py @@ -16,7 +16,7 @@ def test_discover_empty_folder(tmp_path): app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - CURRENT.discover_exe(app_data, cache, prefix=str(tmp_path)) + CURRENT.discover_exe(app_data, prefix=str(tmp_path), cache=cache) BASE = (CURRENT.install_path("scripts"), ".") @@ -43,7 +43,7 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog): # no if pyvenv.exists(): (folder / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") inside_folder = str(tmp_path) - base = CURRENT.discover_exe(app_data, cache, inside_folder) + base = CURRENT.discover_exe(app_data, inside_folder, cache=cache) found = base.executable dest_str = str(dest) if not fs_is_case_sensitive(): @@ -56,4 +56,4 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog): # no dest.rename(dest.parent / (dest.name + "-1")) CURRENT._cache_exe_discovery.clear() # noqa: SLF001 with pytest.raises(RuntimeError): - CURRENT.discover_exe(app_data, cache, inside_folder) + CURRENT.discover_exe(app_data, inside_folder, cache=cache) From 78a8dc2d1262eef0e48da8dc153f492a6e26f198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Thu, 9 Oct 2025 15:20:35 -0700 Subject: [PATCH 13/17] release 20.35.1 --- docs/changelog.rst | 8 ++++++++ docs/changelog/2972.bugfix.rst | 1 - docs/changelog/2975.bugfix.rst | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 docs/changelog/2972.bugfix.rst delete mode 100644 docs/changelog/2975.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index c37eda600..638e43f46 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,14 @@ Release History .. towncrier release notes start +v20.35.1 (2025-10-09) +--------------------- + +Bugfixes - 20.35.1 +~~~~~~~~~~~~~~~~~~ +- Patch get_interpreter to handle missing cache and app_data - by :user:`esafak` (:issue:`2972`) +- Fix backwards incompatible changes to ``PythonInfo`` - by :user:`gaborbernat`. (:issue:`2975`) + v20.35.0 (2025-10-08) --------------------- diff --git a/docs/changelog/2972.bugfix.rst b/docs/changelog/2972.bugfix.rst deleted file mode 100644 index 5e38f237e..000000000 --- a/docs/changelog/2972.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Patch get_interpreter to handle missing cache and app_data - by :user:`esafak` diff --git a/docs/changelog/2975.bugfix.rst b/docs/changelog/2975.bugfix.rst deleted file mode 100644 index 87f3ba69a..000000000 --- a/docs/changelog/2975.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix backwards incompatible changes to ``PythonInfo`` - by :user:`gaborbernat`. From 9b47ce9616bbac8379a6cb682b4429be935ed515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 10 Oct 2025 09:52:14 -0700 Subject: [PATCH 14/17] Merge pull request #2978 from gaborbernat/2977 --- docs/changelog/2978.bugfix.rst | 1 + src/virtualenv/cache/__init__.py | 9 - src/virtualenv/cache/cache.py | 62 --- src/virtualenv/cache/file_cache.py | 47 -- src/virtualenv/create/via_global_ref/venv.py | 2 +- src/virtualenv/discovery/app_data.py | 23 - src/virtualenv/discovery/builtin.py | 53 +-- src/virtualenv/discovery/cache.py | 18 - src/virtualenv/discovery/cached_py_info.py | 120 +++-- src/virtualenv/discovery/discover.py | 3 +- src/virtualenv/discovery/info.py | 33 -- src/virtualenv/discovery/py_info.py | 55 +-- src/virtualenv/discovery/windows/__init__.py | 10 +- src/virtualenv/run/__init__.py | 3 - src/virtualenv/run/plugin/discovery.py | 2 +- tests/__init__.py | 0 tests/conftest.py | 13 +- tests/integration/test_zipapp.py | 14 +- tests/unit/activation/conftest.py | 13 +- tests/unit/config/test_env_var.py | 7 +- tests/unit/create/conftest.py | 24 +- tests/unit/create/test_creator.py | 424 ++++++++---------- tests/unit/create/test_interpreters.py | 24 +- .../create/via_global_ref/test_build_c_ext.py | 114 +++-- tests/unit/discovery/py_info/test_py_info.py | 68 ++- .../py_info/test_py_info_exe_based_of.py | 15 +- tests/unit/discovery/test_discovery.py | 91 ++-- tests/unit/discovery/util.py | 51 --- tests/unit/discovery/windows/test_windows.py | 3 +- .../embed/test_bootstrap_link_via_app_data.py | 13 +- tests/unit/test_file_limit.py | 24 +- 31 files changed, 440 insertions(+), 899 deletions(-) create mode 100644 docs/changelog/2978.bugfix.rst delete mode 100644 src/virtualenv/cache/__init__.py delete mode 100644 src/virtualenv/cache/cache.py delete mode 100644 src/virtualenv/cache/file_cache.py delete mode 100644 src/virtualenv/discovery/app_data.py delete mode 100644 src/virtualenv/discovery/cache.py delete mode 100644 src/virtualenv/discovery/info.py delete mode 100644 tests/__init__.py delete mode 100644 tests/unit/discovery/util.py diff --git a/docs/changelog/2978.bugfix.rst b/docs/changelog/2978.bugfix.rst new file mode 100644 index 000000000..013023fe3 --- /dev/null +++ b/docs/changelog/2978.bugfix.rst @@ -0,0 +1 @@ +Revert out changes related to the extraction of the discovery module - by :user:`gaborbernat`. diff --git a/src/virtualenv/cache/__init__.py b/src/virtualenv/cache/__init__.py deleted file mode 100644 index 8f929c60c..000000000 --- a/src/virtualenv/cache/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -from .cache import Cache -from .file_cache import FileCache - -__all__ = [ - "Cache", - "FileCache", -] diff --git a/src/virtualenv/cache/cache.py b/src/virtualenv/cache/cache.py deleted file mode 100644 index a13df5aaa..000000000 --- a/src/virtualenv/cache/cache.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any, Generic, Hashable, TypeVar - -try: - from typing import Self # pragma: ≥ 3.11 cover -except ImportError: - from typing_extensions import Self # pragma: < 3.11 cover - -K = TypeVar("K", bound=Hashable) - - -class Cache(ABC, Generic[K]): - """ - A generic cache interface. - - Add a close() method if the cache needs to perform any cleanup actions, - and an __exit__ method to allow it to be used in a context manager. - """ - - @abstractmethod - def get(self, key: K) -> Any | None: - """ - Get a value from the cache. - - :param key: the key to retrieve - :return: the cached value, or None if not found - """ - raise NotImplementedError - - @abstractmethod - def set(self, key: K, value: Any) -> None: - """ - Set a value in the cache. - - :param key: the key to set - :param value: the value to cache - """ - raise NotImplementedError - - @abstractmethod - def remove(self, key: K) -> None: - """ - Remove a value from the cache. - - :param key: the key to remove - """ - raise NotImplementedError - - @abstractmethod - def clear(self) -> None: - """Clear the entire cache.""" - raise NotImplementedError - - def __enter__(self) -> Self: - return self - - -__all__ = [ - "Cache", -] diff --git a/src/virtualenv/cache/file_cache.py b/src/virtualenv/cache/file_cache.py deleted file mode 100644 index 2a5605aea..000000000 --- a/src/virtualenv/cache/file_cache.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Callable - -from virtualenv.cache import Cache - -if TYPE_CHECKING: - from pathlib import Path - - from virtualenv.app_data.base import ContentStore - - -class FileCache(Cache): - def __init__(self, store_factory: Callable[[Path], ContentStore], clearer: Callable[[], None] | None) -> None: - self.store_factory = store_factory - self.clearer = clearer - - def get(self, key: Path) -> dict | None: - """Get a value from the file cache.""" - result, store = None, self.store_factory(key) - with store.locked(): - if store.exists(): - result = store.read() - return result - - def set(self, key: Path, value: dict) -> None: - """Set a value in the file cache.""" - store = self.store_factory(key) - with store.locked(): - store.write(value) - - def remove(self, key: Path) -> None: - """Remove a value from the file cache.""" - store = self.store_factory(key) - with store.locked(): - if store.exists(): - store.remove() - - def clear(self) -> None: - """Clear the entire file cache.""" - if self.clearer is not None: - self.clearer() - - -__all__ = [ - "FileCache", -] diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py index 45866d676..d5037bc3b 100644 --- a/src/virtualenv/create/via_global_ref/venv.py +++ b/src/virtualenv/create/via_global_ref/venv.py @@ -20,7 +20,7 @@ class Venv(ViaGlobalRefApi): def __init__(self, options, interpreter) -> None: self.describe = options.describe super().__init__(options, interpreter) - current = PythonInfo.current(options.app_data, options.cache) + current = PythonInfo.current() self.can_be_inline = interpreter is current and interpreter.executable == interpreter.system_executable self._context = None diff --git a/src/virtualenv/discovery/app_data.py b/src/virtualenv/discovery/app_data.py deleted file mode 100644 index 90de58284..000000000 --- a/src/virtualenv/discovery/app_data.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, ContextManager, Protocol - -if TYPE_CHECKING: - from pathlib import Path - - -class AppData(Protocol): - """Protocol for application data store.""" - - def py_info(self, path: Path) -> Any: ... - - def py_info_clear(self) -> None: ... - - @contextmanager - def ensure_extracted(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: ... - - @contextmanager - def extract(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: ... - - def close(self) -> None: ... diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index 1e4364db5..e2d193911 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -9,8 +9,9 @@ from platformdirs import user_data_path +from virtualenv.info import IS_WIN, fs_path_id + from .discover import Discover -from .info import IS_WIN, fs_path_id from .py_info import PythonInfo from .py_spec import PythonSpec @@ -18,7 +19,7 @@ from argparse import ArgumentParser from collections.abc import Callable, Generator, Iterable, Mapping, Sequence - from .app_data import AppData + from virtualenv.app_data.base import AppData LOGGER = logging.getLogger(__name__) @@ -27,8 +28,8 @@ class Builtin(Discover): app_data: AppData try_first_with: Sequence[str] - def __init__(self, options, cache=None) -> None: - super().__init__(options, cache) + def __init__(self, options) -> None: + super().__init__(options) self.python_spec = options.python or [sys.executable] if self._env.get("VIRTUALENV_PYTHON"): self.python_spec = self.python_spec[1:] + self.python_spec[:1] # Rotate the list @@ -60,7 +61,7 @@ def add_parser_arguments(cls, parser: ArgumentParser) -> None: def run(self) -> PythonInfo | None: for python_spec in self.python_spec: - result = get_interpreter(python_spec, self.try_first_with, self.app_data, self.cache, self._env) + result = get_interpreter(python_spec, self.try_first_with, self.app_data, self._env) if result is not None: return result return None @@ -71,36 +72,13 @@ def __repr__(self) -> str: def get_interpreter( - key, - try_first_with: Iterable[str], - app_data: AppData | None = None, - cache=None, - env: Mapping[str, str] | None = None, + key, try_first_with: Iterable[str], app_data: AppData | None = None, env: Mapping[str, str] | None = None ) -> PythonInfo | None: - """ - Find an interpreter that matches a given specification. - - :param key: the specification of the interpreter to find - :param try_first_with: a list of interpreters to try first - :param app_data: the application data folder - :param cache: a cache of python information - :param env: the environment to use - :return: the interpreter if found, otherwise None - """ - if cache is None: - # Import locally to avoid a circular dependency - from virtualenv.app_data import AppDataDisabled # noqa: PLC0415 - from virtualenv.cache import FileCache # noqa: PLC0415 - - if app_data is None: - app_data = AppDataDisabled() - cache = FileCache(store_factory=app_data.py_info, clearer=app_data.py_info_clear) - spec = PythonSpec.from_string_spec(key) LOGGER.info("find interpreter for spec %r", spec) proposed_paths = set() env = os.environ if env is None else env - for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, cache, env): + for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env): key = interpreter.system_executable, impl_must_match if key in proposed_paths: continue @@ -116,7 +94,6 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 spec: PythonSpec, try_first_with: Iterable[str], app_data: AppData | None = None, - cache=None, env: Mapping[str, str] | None = None, ) -> Generator[tuple[PythonInfo, bool], None, None]: # 0. if it's a path and exists, and is absolute path, this is the only option we consider @@ -132,7 +109,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 exe_id = fs_path_id(exe_raw) if exe_id not in tested_exes: tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True + yield PythonInfo.from_exe(exe_raw, app_data, env=env), True return # 1. try with first @@ -148,7 +125,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 if exe_id in tested_exes: continue tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True + yield PythonInfo.from_exe(exe_raw, app_data, env=env), True # 1. if it's a path and exists if spec.path is not None: @@ -161,12 +138,12 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 exe_id = fs_path_id(exe_raw) if exe_id not in tested_exes: tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True + yield PythonInfo.from_exe(exe_raw, app_data, env=env), True if spec.is_abs: return else: # 2. otherwise try with the current - current_python = PythonInfo.current_system(app_data, cache) + current_python = PythonInfo.current_system(app_data) exe_raw = str(current_python.executable) exe_id = fs_path_id(exe_raw) if exe_id not in tested_exes: @@ -177,7 +154,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 if IS_WIN: from .windows import propose_interpreters # noqa: PLC0415 - for interpreter in propose_interpreters(spec, app_data, cache, env): + for interpreter in propose_interpreters(spec, app_data, env): exe_raw = str(interpreter.executable) exe_id = fs_path_id(exe_raw) if exe_id in tested_exes: @@ -195,7 +172,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 if exe_id in tested_exes: continue tested_exes.add(exe_id) - interpreter = PathPythonInfo.from_exe(exe_raw, app_data, cache, raise_on_error=False, env=env) + interpreter = PathPythonInfo.from_exe(exe_raw, app_data, raise_on_error=False, env=env) if interpreter is not None: yield interpreter, impl_must_match @@ -208,7 +185,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 uv_python_path = user_data_path("uv") / "python" for exe_path in uv_python_path.glob("*/bin/python"): - interpreter = PathPythonInfo.from_exe(str(exe_path), app_data, cache, raise_on_error=False, env=env) + interpreter = PathPythonInfo.from_exe(str(exe_path), app_data, raise_on_error=False, env=env) if interpreter is not None: yield interpreter, True diff --git a/src/virtualenv/discovery/cache.py b/src/virtualenv/discovery/cache.py deleted file mode 100644 index eaf24cc14..000000000 --- a/src/virtualenv/discovery/cache.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Protocol - -if TYPE_CHECKING: - from pathlib import Path - - -class Cache(Protocol): - """A protocol for a cache.""" - - def get(self, path: Path) -> Any: ... - - def set(self, path: Path, data: Any) -> None: ... - - def remove(self, path: Path) -> None: ... - - def clear(self) -> None: ... diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index 645a5eb36..ee2034615 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -8,41 +8,28 @@ from __future__ import annotations import hashlib -import importlib.util import logging import os import random -import subprocess import sys from collections import OrderedDict from pathlib import Path from shlex import quote from string import ascii_lowercase, ascii_uppercase, digits -from typing import TYPE_CHECKING +from subprocess import Popen -from .py_info import PythonInfo - -if TYPE_CHECKING: - from .app_data import AppData - from .cache import Cache +from virtualenv.app_data import AppDataDisabled +from virtualenv.discovery.py_info import PythonInfo +from virtualenv.util.subprocess import subprocess _CACHE = OrderedDict() _CACHE[Path(sys.executable)] = PythonInfo() LOGGER = logging.getLogger(__name__) -def from_exe( # noqa: PLR0913 - cls, - app_data: AppData, - exe: str, - env: dict[str, str] | None = None, - *, - raise_on_error: bool = True, - ignore_cache: bool = False, - cache: Cache, -) -> PythonInfo | None: +def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=False): # noqa: FBT002, PLR0913 env = os.environ if env is None else env - result = _get_from_cache(cls, app_data, exe, env, cache, ignore_cache=ignore_cache) + result = _get_from_cache(cls, app_data, exe, env, ignore_cache=ignore_cache) if isinstance(result, Exception): if raise_on_error: raise result @@ -51,59 +38,63 @@ def from_exe( # noqa: PLR0913 return result -def _get_from_cache(cls, app_data: AppData, exe: str, env, cache: Cache, *, ignore_cache: bool) -> PythonInfo: # noqa: PLR0913 +def _get_from_cache(cls, app_data, exe, env, ignore_cache=True): # noqa: FBT002 # note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a # pyenv.cfg somewhere alongside on python3.5+ exe_path = Path(exe) if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache result = _CACHE[exe_path] else: # otherwise go through the app data cache - result = _CACHE[exe_path] = _get_via_file_cache(cls, app_data, exe_path, exe, env, cache) + py_info = _get_via_file_cache(cls, app_data, exe_path, exe, env) + result = _CACHE[exe_path] = py_info # independent if it was from the file or in-memory cache fix the original executable location if isinstance(result, PythonInfo): result.executable = exe return result -def _get_via_file_cache(cls, app_data: AppData, path: Path, exe: str, env, cache: Cache) -> PythonInfo: # noqa: PLR0913 - # 1. get the hash of the probing script - spec = importlib.util.find_spec("virtualenv.discovery.py_info") - script = Path(spec.origin) - try: - py_info_hash = hashlib.sha256(script.read_bytes()).hexdigest() - except OSError: - py_info_hash = None - - # 2. get the mtime of the python executable +def _get_via_file_cache(cls, app_data, path, exe, env): + path_text = str(path) try: path_modified = path.stat().st_mtime except OSError: path_modified = -1 + py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" + try: + py_info_hash = hashlib.sha256(py_info_script.read_bytes()).hexdigest() + except OSError: + py_info_hash = None - # 3. check if we have a valid cache entry - py_info = None - data = cache.get(path) - if data is not None: - if data.get("path") == str(path) and data.get("st_mtime") == path_modified and data.get("hash") == py_info_hash: - py_info = cls._from_dict(data.get("content")) - sys_exe = py_info.system_executable - if sys_exe is not None and not os.path.exists(sys_exe): - py_info = None # if system executable is no longer there, this is not valid - if py_info is None: - cache.remove(path) # if cache is invalid, remove it - - if py_info is None: # if not loaded run and save - failure, py_info = _run_subprocess(cls, exe, app_data, env) - if failure is None: - data = { - "st_mtime": path_modified, - "path": str(path), - "content": py_info._to_dict(), # noqa: SLF001 - "hash": py_info_hash, - } - cache.set(path, data) - else: - py_info = failure + if app_data is None: + app_data = AppDataDisabled() + py_info, py_info_store = None, app_data.py_info(path) + with py_info_store.locked(): + if py_info_store.exists(): # if exists and matches load + data = py_info_store.read() + of_path = data.get("path") + of_st_mtime = data.get("st_mtime") + of_content = data.get("content") + of_hash = data.get("hash") + if of_path == path_text and of_st_mtime == path_modified and of_hash == py_info_hash: + py_info = cls._from_dict(of_content.copy()) + sys_exe = py_info.system_executable + if sys_exe is not None and not os.path.exists(sys_exe): + py_info_store.remove() + py_info = None + else: + py_info_store.remove() + if py_info is None: # if not loaded run and save + failure, py_info = _run_subprocess(cls, exe, app_data, env) + if failure is None: + data = { + "st_mtime": path_modified, + "path": path_text, + "content": py_info._to_dict(), # noqa: SLF001 + "hash": py_info_hash, + } + py_info_store.write(data) + else: + py_info = failure return py_info @@ -117,12 +108,7 @@ def gen_cookie(): ) -def _run_subprocess( - cls, - exe: str, - app_data: AppData, - env: dict[str, str], -) -> tuple[Exception | None, PythonInfo | None]: +def _run_subprocess(cls, exe, app_data, env): py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" # Cookies allow to split the serialized stdout output generated by the script collecting the info from the output # generated by something else. The right way to deal with it is to create an anonymous pipe and pass its descriptor @@ -134,14 +120,14 @@ def _run_subprocess( start_cookie = gen_cookie() end_cookie = gen_cookie() - with app_data.ensure_extracted(py_info_script) as py_info_script_path: - cmd = [exe, str(py_info_script_path), start_cookie, end_cookie] + with app_data.ensure_extracted(py_info_script) as py_info_script: + cmd = [exe, str(py_info_script), start_cookie, end_cookie] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = env.copy() env.pop("__PYVENV_LAUNCHER__", None) LOGGER.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: - process = subprocess.Popen( + process = Popen( cmd, universal_newlines=True, stdin=subprocess.PIPE, @@ -196,10 +182,8 @@ def __repr__(self) -> str: return cmd_repr -def clear(cache: Cache | None = None) -> None: - """Clear the cache.""" - if cache is not None: - cache.clear() +def clear(app_data): + app_data.py_info_clear() _CACHE.clear() diff --git a/src/virtualenv/discovery/discover.py b/src/virtualenv/discovery/discover.py index de1b5fd0b..0aaa17c8e 100644 --- a/src/virtualenv/discovery/discover.py +++ b/src/virtualenv/discovery/discover.py @@ -15,7 +15,7 @@ def add_parser_arguments(cls, parser): """ raise NotImplementedError - def __init__(self, options, cache=None) -> None: + def __init__(self, options) -> None: """ Create a new discovery mechanism. @@ -24,7 +24,6 @@ def __init__(self, options, cache=None) -> None: self._has_run = False self._interpreter = None self._env = options.env - self.cache = cache @abstractmethod def run(self): diff --git a/src/virtualenv/discovery/info.py b/src/virtualenv/discovery/info.py deleted file mode 100644 index c786a6a98..000000000 --- a/src/virtualenv/discovery/info.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -import logging -import os -import sys -import tempfile - -_FS_CASE_SENSITIVE = None -LOGGER = logging.getLogger(__name__) -IS_WIN = sys.platform == "win32" - - -def fs_is_case_sensitive(): - """Check if the file system is case-sensitive.""" - global _FS_CASE_SENSITIVE # noqa: PLW0603 - - if _FS_CASE_SENSITIVE is None: - with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: - _FS_CASE_SENSITIVE = not os.path.exists(tmp_file.name.lower()) - LOGGER.debug("filesystem is %scase-sensitive", "" if _FS_CASE_SENSITIVE else "not ") - return _FS_CASE_SENSITIVE - - -def fs_path_id(path: str) -> str: - """Get a case-normalized path identifier.""" - return path.casefold() if fs_is_case_sensitive() else path - - -__all__ = ( - "IS_WIN", - "fs_is_case_sensitive", - "fs_path_id", -) diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 960421c30..c2310cd7e 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -315,7 +315,7 @@ def creators(self, refresh=False): # noqa: FBT002 if self._creators is None or refresh is True: from virtualenv.run.plugin.creators import CreatorSelector # noqa: PLC0415 - self._creators = CreatorSelector.for_interpreter(self) + self._creators = CreatorSelector.for_interpreter(self) return self._creators @property @@ -386,11 +386,11 @@ def spec(self): ) @classmethod - def clear_cache(cls, cache=None): + def clear_cache(cls, app_data): # this method is not used by itself, so here and called functions can import stuff locally from virtualenv.discovery.cached_py_info import clear # noqa: PLC0415 - clear(cache) + clear(app_data) cls._cache_exe_discovery.clear() def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 @@ -431,35 +431,23 @@ def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 _current = None @classmethod - def current(cls, app_data=None, cache=None): + def current(cls, app_data=None): """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. """ # noqa: D205 if cls._current is None: - cls._current = cls.from_exe( - sys.executable, - app_data, - cache, - raise_on_error=True, - resolve_to_host=False, - ) + cls._current = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=False) return cls._current @classmethod - def current_system(cls, app_data=None, cache=None) -> PythonInfo: + def current_system(cls, app_data=None) -> PythonInfo: """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. """ # noqa: D205 if cls._current_system is None: - cls._current_system = cls.from_exe( - sys.executable, - app_data, - cache, - raise_on_error=True, - resolve_to_host=True, - ) + cls._current_system = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=True) return cls._current_system def _to_json(self): @@ -467,7 +455,8 @@ def _to_json(self): return json.dumps(self._to_dict(), indent=2) def _to_dict(self): - data = {var: getattr(self, var) for var in vars(self)} + data = {var: (getattr(self, var) if var != "_creators" else None) for var in vars(self)} + data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary return data @@ -476,7 +465,6 @@ def from_exe( # noqa: PLR0913 cls, exe, app_data=None, - cache=None, raise_on_error=True, # noqa: FBT002 ignore_cache=False, # noqa: FBT002 resolve_to_host=True, # noqa: FBT002 @@ -484,16 +472,14 @@ def from_exe( # noqa: PLR0913 ): """Given a path to an executable get the python information.""" # this method is not used by itself, so here and called functions can import stuff locally - from virtualenv.discovery.cached_py_info import from_exe as from_exe_cache # noqa: PLC0415 + from virtualenv.discovery.cached_py_info import from_exe # noqa: PLC0415 env = os.environ if env is None else env - proposed = from_exe_cache( - cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache, cache=cache - ) + proposed = from_exe(cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache) if isinstance(proposed, PythonInfo) and resolve_to_host: try: - proposed = proposed._resolve_to_system(app_data, proposed, cache=cache) # noqa: SLF001 + proposed = proposed._resolve_to_system(app_data, proposed) # noqa: SLF001 except Exception as exception: if raise_on_error: raise @@ -515,7 +501,7 @@ def _from_dict(cls, data): return result @classmethod - def _resolve_to_system(cls, app_data, target, cache): + def _resolve_to_system(cls, app_data, target): start_executable = target.executable prefixes = OrderedDict() while target.system_executable is None: @@ -532,15 +518,15 @@ def _resolve_to_system(cls, app_data, target, cache): msg = "prefixes are causing a circle {}".format("|".join(prefixes.keys())) raise RuntimeError(msg) prefixes[prefix] = target - target = target.discover_exe(app_data, prefix=prefix, exact=False, cache=cache) + target = target.discover_exe(app_data, prefix=prefix, exact=False) if target.executable != target.system_executable: - target = cls.from_exe(target.system_executable, app_data, cache) + target = cls.from_exe(target.system_executable, app_data) target.executable = start_executable return target _cache_exe_discovery = {} # noqa: RUF012 - def discover_exe(self, app_data, prefix, exact=True, env=None, cache=None): # noqa: FBT002 + def discover_exe(self, app_data, prefix, exact=True, env=None): # noqa: FBT002 key = prefix, exact if key in self._cache_exe_discovery and prefix: LOGGER.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) @@ -553,7 +539,7 @@ def discover_exe(self, app_data, prefix, exact=True, env=None, cache=None): # n env = os.environ if env is None else env for folder in possible_folders: for name in possible_names: - info = self._check_exe(app_data, cache, folder, name, exact, discovered, env) + info = self._check_exe(app_data, folder, name, exact, discovered, env) if info is not None: self._cache_exe_discovery[key] = info return info @@ -566,11 +552,11 @@ def discover_exe(self, app_data, prefix, exact=True, env=None, cache=None): # n msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) raise RuntimeError(msg) - def _check_exe(self, app_data, cache, folder, name, exact, discovered, env): # noqa: PLR0913 + def _check_exe(self, app_data, folder, name, exact, discovered, env): # noqa: PLR0913 exe_path = os.path.join(folder, name) if not os.path.exists(exe_path): return None - info = self.from_exe(exe_path, app_data, cache, resolve_to_host=False, raise_on_error=False, env=env) + info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False, env=env) if info is None: # ignore if for some reason we can't query return None for item in ["implementation", "architecture", "version_info"]: @@ -654,8 +640,7 @@ def _possible_base(self): for base in possible_base: lower = base.lower() yield lower - - from virtualenv.discovery.info import fs_is_case_sensitive # noqa: PLC0415 + from virtualenv.info import fs_is_case_sensitive # noqa: PLC0415 if fs_is_case_sensitive(): if base != lower: diff --git a/src/virtualenv/discovery/windows/__init__.py b/src/virtualenv/discovery/windows/__init__.py index ef47a90e3..b7206406a 100644 --- a/src/virtualenv/discovery/windows/__init__.py +++ b/src/virtualenv/discovery/windows/__init__.py @@ -16,7 +16,7 @@ class Pep514PythonInfo(PythonInfo): """A Python information acquired from PEP-514.""" -def propose_interpreters(spec, app_data, cache, env): +def propose_interpreters(spec, cache_dir, env): # see if PEP-514 entries are good # start with higher python versions in an effort to use the latest version available @@ -36,13 +36,7 @@ def propose_interpreters(spec, app_data, cache, env): skip_pre_filter = implementation.lower() != "cpython" registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe, free_threaded=threaded) if skip_pre_filter or registry_spec.satisfies(spec): - interpreter = Pep514PythonInfo.from_exe( - exe, - app_data, - cache, - raise_on_error=False, - env=env, - ) + interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False) if interpreter is not None and interpreter.satisfies(spec, impl_must_match=True): yield interpreter # Final filtering/matching using interpreter metadata diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index 208050c90..03190502b 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -5,7 +5,6 @@ from functools import partial from virtualenv.app_data import make_app_data -from virtualenv.cache import FileCache from virtualenv.config.cli.parser import VirtualEnvConfigParser from virtualenv.report import LEVELS, setup_report from virtualenv.run.session import Session @@ -131,8 +130,6 @@ def load_app_data(args, parser, options): options, _ = parser.parse_known_args(args, namespace=options) if options.reset_app_data: options.app_data.reset() - - options.cache = FileCache(store_factory=options.app_data.py_info, clearer=options.app_data.py_info_clear) return options diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py index b271faac5..5e8b2392f 100644 --- a/src/virtualenv/run/plugin/discovery.py +++ b/src/virtualenv/run/plugin/discovery.py @@ -32,7 +32,7 @@ def get_discover(parser, args): discover_class = discover_types[options.discovery] discover_class.add_parser_arguments(discovery_parser) options, _ = parser.parse_known_args(args, namespace=options) - return discover_class(options, options.cache) + return discover_class(options) def _get_default_discovery(discover_types): diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/conftest.py b/tests/conftest.py index 4fcb25da8..e4fc28479 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,11 +12,9 @@ import pytest from virtualenv.app_data import AppDataDiskFolder -from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_GRAALPY, IS_PYPY, IS_WIN, fs_supports_symlink from virtualenv.report import LOGGER -from virtualenv.run.plugin.creators import CreatorSelector def pytest_addoption(parser): @@ -126,10 +124,9 @@ def _check_cwd_not_changed_by_test(): @pytest.fixture(autouse=True) def _ensure_py_info_cache_empty(session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - PythonInfo.clear_cache(cache) + PythonInfo.clear_cache(session_app_data) yield - PythonInfo.clear_cache(cache) + PythonInfo.clear_cache(session_app_data) @contextmanager @@ -311,8 +308,7 @@ def special_name_dir(tmp_path, special_char_name): @pytest.fixture(scope="session") def current_creators(session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - return CreatorSelector.for_interpreter(PythonInfo.current_system(session_app_data, cache)) + return PythonInfo.current_system(session_app_data).creators() @pytest.fixture(scope="session") @@ -360,8 +356,7 @@ def for_py_version(): @pytest.fixture def _skip_if_test_in_system(session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - current = PythonInfo.current(session_app_data, cache) + current = PythonInfo.current(session_app_data) if current.system_executable is not None: pytest.skip("test not valid if run under system") diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py index e40ecca91..c6c46c7bc 100644 --- a/tests/integration/test_zipapp.py +++ b/tests/integration/test_zipapp.py @@ -7,25 +7,19 @@ import pytest -from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink from virtualenv.run import cli_run HERE = Path(__file__).parent +CURRENT = PythonInfo.current_system() @pytest.fixture(scope="session") -def current_info(session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - return PythonInfo.current_system(session_app_data, cache) - - -@pytest.fixture(scope="session") -def zipapp_build_env(tmp_path_factory, current_info): +def zipapp_build_env(tmp_path_factory): create_env_path = None - if current_info.implementation not in {"PyPy", "GraalVM"}: - exe = current_info.executable # guaranteed to contain a recent enough pip (tox.ini) + if CURRENT.implementation not in {"PyPy", "GraalVM"}: + exe = CURRENT.executable # guaranteed to contain a recent enough pip (tox.ini) else: create_env_path = tmp_path_factory.mktemp("zipapp-create-env") exe, found = None, False diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index a7186896f..53a819f96 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -229,18 +229,9 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session", params=[True, False], ids=["with_prompt", "no_prompt"]) -def activation_python(request, tmp_path_factory, special_char_name, current_fastest, session_app_data): +def activation_python(request, tmp_path_factory, special_char_name, current_fastest): dest = os.path.join(str(tmp_path_factory.mktemp("activation-tester-env")), special_char_name) - cmd = [ - "--without-pip", - dest, - "--creator", - current_fastest, - "-vv", - "--no-periodic-update", - "--app-data", - str(session_app_data), - ] + cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv", "--no-periodic-update"] # `params` is accessed here. https://docs.pytest.org/en/stable/reference/reference.html#pytest-fixture if request.param: cmd += ["--prompt", special_char_name] diff --git a/tests/unit/config/test_env_var.py b/tests/unit/config/test_env_var.py index 8ec16c607..5f364978d 100644 --- a/tests/unit/config/test_env_var.py +++ b/tests/unit/config/test_env_var.py @@ -5,7 +5,6 @@ import pytest -from virtualenv.cache import FileCache from virtualenv.config.cli.parser import VirtualEnvOptions from virtualenv.config.ini import IniConfig from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew @@ -77,12 +76,10 @@ def test_extra_search_dir_via_env_var(tmp_path, monkeypatch): @pytest.mark.usefixtures("_empty_conf") -def test_value_alias(monkeypatch, mocker, session_app_data): +@pytest.mark.skipif(is_macos_brew(PythonInfo.current_system()), reason="no copy on brew") +def test_value_alias(monkeypatch, mocker): from virtualenv.config.cli.parser import VirtualEnvConfigParser # noqa: PLC0415 - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - if is_macos_brew(PythonInfo.current_system(session_app_data, cache)): - pytest.skip(reason="no copy on brew") prev = VirtualEnvConfigParser._fix_default # noqa: SLF001 def func(self, action): diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py index bb897dd12..58d390c5c 100644 --- a/tests/unit/create/conftest.py +++ b/tests/unit/create/conftest.py @@ -14,31 +14,25 @@ import pytest -from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo +CURRENT = PythonInfo.current_system() -@pytest.fixture(scope="session") -def current_info(session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - return PythonInfo.current_system(session_app_data, cache) +def root(tmp_path_factory, session_app_data): # noqa: ARG001 + return CURRENT.system_executable -def root(tmp_path_factory, session_app_data, current_info): # noqa: ARG001 - return current_info.system_executable - -def venv(tmp_path_factory, session_app_data, current_info): - if current_info.is_venv: +def venv(tmp_path_factory, session_app_data): + if CURRENT.is_venv: return sys.executable - root_python = root(tmp_path_factory, session_app_data, current_info) + root_python = root(tmp_path_factory, session_app_data) dest = tmp_path_factory.mktemp("venv") process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)]) process.communicate() # sadly creating a virtual environment does not tell us where the executable lives in general case # so discover using some heuristic - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - return current_info.discover_exe(session_app_data, prefix=str(dest), cache=cache).original_executable + return CURRENT.discover_exe(prefix=str(dest)).original_executable PYTHON = { @@ -48,8 +42,8 @@ def venv(tmp_path_factory, session_app_data, current_info): @pytest.fixture(params=list(PYTHON.values()), ids=list(PYTHON.keys()), scope="session") -def python(request, tmp_path_factory, session_app_data, current_info): - result = request.param(tmp_path_factory, session_app_data, current_info) +def python(request, tmp_path_factory, session_app_data): + result = request.param(tmp_path_factory, session_app_data) if isinstance(result, Exception): pytest.skip(f"could not resolve interpreter based on {request.param.__name__} because {result}") if result is None: diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index 0e80514b1..a80ab16ee 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -14,6 +14,7 @@ import textwrap import zipfile from collections import OrderedDict +from itertools import product from pathlib import Path from stat import S_IREAD, S_IRGRP, S_IROTH from textwrap import dedent @@ -22,7 +23,6 @@ import pytest from virtualenv.__main__ import run, run_with_catch -from virtualenv.cache import FileCache from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info from virtualenv.create.pyenv_cfg import PyEnvCfg from virtualenv.create.via_global_ref import api @@ -31,15 +31,8 @@ from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_PYPY, IS_WIN, fs_is_case_sensitive from virtualenv.run import cli_run, session_via_cli -from virtualenv.run.plugin.creators import CreatorSelector -logger = logging.getLogger(__name__) - - -@pytest.fixture(scope="session") -def current_info(session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - return PythonInfo.current_system(session_app_data, cache) +CURRENT = PythonInfo.current_system() def test_os_path_sep_not_allowed(tmp_path, capsys): @@ -96,179 +89,140 @@ def cleanup_sys_path(paths): @pytest.fixture(scope="session") -def system(session_app_data, current_info): - return get_env_debug_info(Path(current_info.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) +def system(session_app_data): + return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) -@pytest.fixture(scope="session") -def current_creator_keys(current_info): - return [i for i in CreatorSelector.for_interpreter(current_info).key_to_class if i != "builtin"] +CURRENT_CREATORS = [i for i in CURRENT.creators().key_to_class if i != "builtin"] +CREATE_METHODS = [] +for k, v in CURRENT.creators().key_to_meta.items(): + if k in CURRENT_CREATORS: + if v.can_copy: + if k == "venv" and CURRENT.implementation == "PyPy" and CURRENT.pypy_version_info >= [7, 3, 13]: + continue # https://foss.heptapod.net/pypy/pypy/-/issues/4019 + CREATE_METHODS.append((k, "copies")) + if v.can_symlink: + CREATE_METHODS.append((k, "symlinks")) -@pytest.fixture(scope="session") -def create_methods(current_creator_keys, current_info): - methods = [] - for k, v in CreatorSelector.for_interpreter(current_info).key_to_meta.items(): - if k in current_creator_keys: - if v.can_copy: - if ( - k == "venv" - and current_info.implementation == "PyPy" - and current_info.pypy_version_info >= [7, 3, 13] - ): # https://github.com/pypy/pypy/issues/4019 - continue - methods.append((k, "copies")) - if v.can_symlink: - methods.append((k, "symlinks")) - return methods - - -@pytest.fixture -def python_case(request, current_info): - """Resolve the python under test based on a param value.""" - case = request.param - if case == "venv": - # keep the original skip condition - if sys.executable == current_info.system_executable: - pytest.skip("system") - return sys.executable, "venv" - if case == "root": - return current_info.system_executable, "root" - msg = f"unknown python_case: {case}" - raise RuntimeError(msg) - - -@pytest.mark.parametrize("isolated", ["isolated", "global"]) -@pytest.mark.parametrize("python_case", ["venv", "root"], indirect=True) +@pytest.mark.parametrize( + ("creator", "isolated"), + [pytest.param(*i, id=f"{'-'.join(i[0])}-{i[1]}") for i in product(CREATE_METHODS, ["isolated", "global"])], +) def test_create_no_seed( # noqa: C901, PLR0912, PLR0913, PLR0915 + python, + creator, + isolated, system, coverage_env, special_name_dir, - create_methods, - current_info, - session_app_data, - isolated, - python_case, ): - python_exe, python_id = python_case - logger.info("running no seed test for %s-%s", python_id, isolated) - - for creator_key, method in create_methods: - dest = special_name_dir / f"{creator_key}-{method}-{isolated}" - cmd = [ - "-v", - "-v", - "-p", - str(python_exe), - str(dest), - "--without-pip", - "--activators", - "", - "--creator", - creator_key, - f"--{method}", - ] - if isolated == "global": - cmd.append("--system-site-packages") - result = cli_run(cmd) - creator = result.creator - coverage_env() - if IS_PYPY: - # pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits - # force a close of these on system where the limit is low-ish (e.g. MacOS 256) - gc.collect() - purelib = creator.purelib - patch_files = {purelib / f"{'_virtualenv'}.{i}" for i in ("py", "pyc", "pth")} - patch_files.add(purelib / "__pycache__") - content = set(creator.purelib.iterdir()) - patch_files - assert not content, "\n".join(str(i) for i in content) - assert creator.env_name == str(dest.name) - debug = creator.debug - assert "exception" not in debug, f"{debug.get('exception')}\n{debug.get('out')}\n{debug.get('err')}" - sys_path = cleanup_sys_path(debug["sys"]["path"]) - system_sys_path = cleanup_sys_path(system["sys"]["path"]) - our_paths = set(sys_path) - set(system_sys_path) - our_paths_repr = "\n".join(repr(i) for i in our_paths) - - # ensure we have at least one extra path added - assert len(our_paths) >= 1, our_paths_repr - # ensure all additional paths are related to the virtual environment - for path in our_paths: - msg = "\n".join(str(p) for p in system_sys_path) - msg = f"\n{path!s}\ndoes not start with {dest!s}\nhas:\n{msg}" - assert str(path).startswith(str(dest)), msg - # ensure there's at least a site-packages folder as part of the virtual environment added - assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr - - # ensure the global site package is added or not, depending on flag - global_sys_path = system_sys_path[-1] - if isolated == "isolated": - msg = "\n".join(str(j) for j in sys_path) - msg = f"global sys path {global_sys_path!s} is in virtual environment sys path:\n{msg}" - assert global_sys_path not in sys_path, msg - else: - common = [] - for left, right in zip(reversed(system_sys_path), reversed(sys_path)): - if left == right: - common.append(left) - else: - break - - def list_to_str(iterable): - return [str(i) for i in iterable] - - assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path))) - - # test that the python executables in the bin directory are either: - # - files - # - absolute symlinks outside of the venv - # - relative symlinks inside of the venv - if sys.platform == "win32": - exes = ("python.exe",) - else: - exes = ( - "python", - f"python{sys.version_info.major}", - f"python{sys.version_info.major}.{sys.version_info.minor}", - ) - if creator_key == "venv": - # for venv some repackaging does not includes the pythonx.y - exes = exes[:-1] - for exe in exes: - exe_path = creator.bin_dir / exe - assert exe_path.exists(), "\n".join(str(i) for i in creator.bin_dir.iterdir()) - if not exe_path.is_symlink(): # option 1: a real file - continue # it was a file - link = os.readlink(str(exe_path)) - if not os.path.isabs(link): # option 2: a relative symlink - continue - # option 3: an absolute symlink, should point outside the venv - assert not link.startswith(str(creator.dest)) - - if IS_WIN and current_info.implementation == "CPython": - python_w = creator.exe.parent / "pythonw.exe" - assert python_w.exists() - assert python_w.read_bytes() != creator.exe.read_bytes() - - if creator_key != "venv" and CPython3Posix.pyvenv_launch_patch_active( - PythonInfo.from_exe( - python_exe, - session_app_data, - FileCache(session_app_data.py_info, session_app_data.py_info_clear), - ), - ): - result = subprocess.check_output( - [str(creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'], - text=True, - ).strip() - assert result == "None" - - git_ignore = (dest / ".gitignore").read_text(encoding="utf-8") - if creator_key == "venv" and sys.version_info >= (3, 13): - comment = "# Created by venv; see https://docs.python.org/3/library/venv.html" - else: - comment = "# created by virtualenv automatically" - assert git_ignore.splitlines() == [comment, "*"] + dest = special_name_dir + creator_key, method = creator + cmd = [ + "-v", + "-v", + "-p", + str(python), + str(dest), + "--without-pip", + "--activators", + "", + "--creator", + creator_key, + f"--{method}", + ] + if isolated == "global": + cmd.append("--system-site-packages") + result = cli_run(cmd) + creator = result.creator + coverage_env() + if IS_PYPY: + # pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits + # force a close of these on system where the limit is low-ish (e.g. MacOS 256) + gc.collect() + purelib = creator.purelib + patch_files = {purelib / f"{'_virtualenv'}.{i}" for i in ("py", "pyc", "pth")} + patch_files.add(purelib / "__pycache__") + content = set(creator.purelib.iterdir()) - patch_files + assert not content, "\n".join(str(i) for i in content) + assert creator.env_name == str(dest.name) + debug = creator.debug + assert "exception" not in debug, f"{debug.get('exception')}\n{debug.get('out')}\n{debug.get('err')}" + sys_path = cleanup_sys_path(debug["sys"]["path"]) + system_sys_path = cleanup_sys_path(system["sys"]["path"]) + our_paths = set(sys_path) - set(system_sys_path) + our_paths_repr = "\n".join(repr(i) for i in our_paths) + + # ensure we have at least one extra path added + assert len(our_paths) >= 1, our_paths_repr + # ensure all additional paths are related to the virtual environment + for path in our_paths: + msg = "\n".join(str(p) for p in system_sys_path) + msg = f"\n{path!s}\ndoes not start with {dest!s}\nhas:\n{msg}" + assert str(path).startswith(str(dest)), msg + # ensure there's at least a site-packages folder as part of the virtual environment added + assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr + + # ensure the global site package is added or not, depending on flag + global_sys_path = system_sys_path[-1] + if isolated == "isolated": + msg = "\n".join(str(j) for j in sys_path) + msg = f"global sys path {global_sys_path!s} is in virtual environment sys path:\n{msg}" + assert global_sys_path not in sys_path, msg + else: + common = [] + for left, right in zip(reversed(system_sys_path), reversed(sys_path)): + if left == right: + common.append(left) + else: + break + + def list_to_str(iterable): + return [str(i) for i in iterable] + + assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path))) + + # test that the python executables in the bin directory are either: + # - files + # - absolute symlinks outside of the venv + # - relative symlinks inside of the venv + if sys.platform == "win32": + exes = ("python.exe",) + else: + exes = ("python", f"python{sys.version_info.major}", f"python{sys.version_info.major}.{sys.version_info.minor}") + if creator_key == "venv": + # for venv some repackaging does not includes the pythonx.y + exes = exes[:-1] + for exe in exes: + exe_path = creator.bin_dir / exe + assert exe_path.exists(), "\n".join(str(i) for i in creator.bin_dir.iterdir()) + if not exe_path.is_symlink(): # option 1: a real file + continue # it was a file + link = os.readlink(str(exe_path)) + if not os.path.isabs(link): # option 2: a relative symlink + continue + # option 3: an absolute symlink, should point outside the venv + assert not link.startswith(str(creator.dest)) + + if IS_WIN and CURRENT.implementation == "CPython": + python_w = creator.exe.parent / "pythonw.exe" + assert python_w.exists() + assert python_w.read_bytes() != creator.exe.read_bytes() + + if CPython3Posix.pyvenv_launch_patch_active(PythonInfo.from_exe(python)) and creator_key != "venv": + result = subprocess.check_output( + [str(creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'], + text=True, + ).strip() + assert result == "None" + + git_ignore = (dest / ".gitignore").read_text(encoding="utf-8") + if creator_key == "venv" and sys.version_info >= (3, 13): + comment = "# Created by venv; see https://docs.python.org/3/library/venv.html" + else: + comment = "# created by virtualenv automatically" + assert git_ignore.splitlines() == [comment, "*"] def test_create_cachedir_tag(tmp_path): @@ -318,9 +272,8 @@ def test_create_vcs_ignore_exists_override(tmp_path): assert git_ignore.read_text(encoding="utf-8") == "magic" -def test_venv_fails_not_inline(tmp_path, capsys, mocker, current_info): - if not current_info.has_venv: - pytest.skip("requires interpreter with venv") +@pytest.mark.skipif(not CURRENT.has_venv, reason="requires interpreter with venv") +def test_venv_fails_not_inline(tmp_path, capsys, mocker): if hasattr(os, "geteuid") and os.geteuid() == 0: pytest.skip("no way to check permission restriction when running under root") @@ -336,7 +289,7 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): cfg = str(cfg_path) try: os.chmod(cfg, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH) - cmd = ["-p", str(current_info.executable), str(tmp_path), "--without-pip", "--creator", "venv"] + cmd = ["-p", str(CURRENT.executable), str(tmp_path), "--without-pip", "--creator", "venv"] with pytest.raises(SystemExit) as context: run(cmd) assert context.value.code != 0 @@ -347,45 +300,46 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): assert "Error:" in err, err +@pytest.mark.parametrize("creator", CURRENT_CREATORS) @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"]) -def test_create_clear_resets(tmp_path, clear, caplog, current_creator_keys): +def test_create_clear_resets(tmp_path, creator, clear, caplog): caplog.set_level(logging.DEBUG) - for creator in current_creator_keys: - if creator == "venv" and clear is False: - pytest.skip("venv without clear might fail") - marker = tmp_path / creator / "magic" - cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator, "-vvv"] - cli_run(cmd) + if creator == "venv" and clear is False: + pytest.skip("venv without clear might fail") + marker = tmp_path / "magic" + cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator, "-vvv"] + cli_run(cmd) - marker.write_text("", encoding="utf-8") - assert marker.exists() + marker.write_text("", encoding="utf-8") # if we a marker file this should be gone on a clear run, remain otherwise + assert marker.exists() - cli_run(cmd + (["--clear"] if clear else [])) - assert marker.exists() is not clear + cli_run(cmd + (["--clear"] if clear else [])) + assert marker.exists() is not clear +@pytest.mark.parametrize("creator", CURRENT_CREATORS) @pytest.mark.parametrize("prompt", [None, "magic"]) -def test_prompt_set(tmp_path, prompt, current_creator_keys): - for creator in current_creator_keys: - cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator] - if prompt is not None: - cmd.extend(["--prompt", "magic"]) - - result = cli_run(cmd) - actual_prompt = tmp_path.name if prompt is None else prompt - cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) - if prompt is None: - assert "prompt" not in cfg - elif creator != "venv": - assert "prompt" in cfg, list(cfg.content.keys()) - assert cfg["prompt"] == actual_prompt - - -def test_home_path_is_exe_parent(tmp_path, current_creator_keys): - for creator in current_creator_keys: - cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator] - result = cli_run(cmd) - cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) +def test_prompt_set(tmp_path, creator, prompt): + cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] + if prompt is not None: + cmd.extend(["--prompt", "magic"]) + + result = cli_run(cmd) + actual_prompt = tmp_path.name if prompt is None else prompt + cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) + if prompt is None: + assert "prompt" not in cfg + elif creator != "venv": + assert "prompt" in cfg, list(cfg.content.keys()) + assert cfg["prompt"] == actual_prompt + + +@pytest.mark.parametrize("creator", CURRENT_CREATORS) +def test_home_path_is_exe_parent(tmp_path, creator): + cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] + + result = cli_run(cmd) + cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) # Cannot assume "home" path is a specific value as path resolution may change # between versions (symlinks, framework paths, etc) but we can check that a @@ -446,20 +400,22 @@ def test_create_long_path(tmp_path): @pytest.mark.slow -def test_create_distutils_cfg(tmp_path, monkeypatch, current_creator_keys): - for creator in current_creator_keys: - result = cli_run( - [ - str(tmp_path / creator / "venv"), - "--activators", - "", - "--creator", - creator, - "--setuptools", - "bundle", - ], - ) - app = Path(__file__).parent / "console_app" +@pytest.mark.parametrize("creator", sorted(set(PythonInfo.current_system().creators().key_to_class) - {"builtin"})) +@pytest.mark.usefixtures("session_app_data") +def test_create_distutils_cfg(creator, tmp_path, monkeypatch): + result = cli_run( + [ + str(tmp_path / "venv"), + "--activators", + "", + "--creator", + creator, + "--setuptools", + "bundle", + ], + ) + + app = Path(__file__).parent / "console_app" dest = tmp_path / "console_app" shutil.copytree(str(app), str(dest)) @@ -508,10 +464,9 @@ def list_files(path): return result +@pytest.mark.skipif(is_macos_brew(CURRENT), reason="no copy on brew") @pytest.mark.skip(reason="https://github.com/pypa/setuptools/issues/4640") -def test_zip_importer_can_import_setuptools(tmp_path, current_info): - if is_macos_brew(current_info): - pytest.skip("no copy on brew") +def test_zip_importer_can_import_setuptools(tmp_path): """We're patching the loaders so might fail on r/o loaders, such as zipimporter on CPython<3.8""" result = cli_run( [str(tmp_path / "venv"), "--activators", "", "--no-pip", "--no-wheel", "--copies", "--setuptools", "bundle"], @@ -720,9 +675,8 @@ def _get_sys_path(flag=None): # (specifically venv scripts delivered with Python itself) are not writable. # # https://github.com/pypa/virtualenv/issues/2419 -def test_venv_creator_without_write_perms(tmp_path, mocker, current_creator_keys): - if "venv" not in current_creator_keys: - pytest.skip("test needs venv creator") +@pytest.mark.skipif("venv" not in CURRENT_CREATORS, reason="test needs venv creator") +def test_venv_creator_without_write_perms(tmp_path, mocker): from virtualenv.run.session import Session # noqa: PLC0415 prev = Session._create # noqa: SLF001 @@ -739,10 +693,9 @@ def func(self): cli_run(cmd) -def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker, session_app_data): +def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker): """Test that creating a virtual environment falls back to copies when filesystem has no symlink support.""" - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - if is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): + if is_macos_brew(PythonInfo.from_exe(python)): pytest.skip("brew python on darwin may not support copies, which is tested separately") # Given a filesystem that does not support symlinks @@ -765,14 +718,13 @@ def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker, ses assert result.creator.symlinks is False -def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker, session_app_data): +def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker): """Test that virtualenv fails gracefully when no creation method is supported.""" # Given a filesystem that does not support symlinks mocker.patch("virtualenv.create.via_global_ref.api.fs_supports_symlink", return_value=False) - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) # And a creator that does not support copying - if not is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): + if not is_macos_brew(PythonInfo.from_exe(python)): original_init = api.ViaGlobalRefMeta.__init__ def new_init(self, *args, **kwargs): @@ -795,7 +747,7 @@ def new_init(self, *args, **kwargs): # Then a RuntimeError should be raised with a detailed message assert "neither symlink or copy method supported" in str(excinfo.value) assert "symlink: the filesystem does not supports symlink" in str(excinfo.value) - if is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): + if is_macos_brew(PythonInfo.from_exe(python)): assert "copy: Brew disables copy creation" in str(excinfo.value) else: assert "copy: copying is not supported" in str(excinfo.value) diff --git a/tests/unit/create/test_interpreters.py b/tests/unit/create/test_interpreters.py index b842891f1..ae4452b13 100644 --- a/tests/unit/create/test_interpreters.py +++ b/tests/unit/create/test_interpreters.py @@ -5,7 +5,6 @@ import pytest -from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run @@ -19,14 +18,15 @@ def test_failed_to_find_bad_spec(): assert repr(context.value) == msg -def test_failed_to_find_implementation(mocker, session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - system = PythonInfo.current_system(session_app_data, cache) - of_ids = ({sys.executable} if sys.executable != system.executable else set()) | {system.implementation} - for of_id in of_ids: - mocker.patch("virtualenv.run.plugin.creators.CreatorSelector._OPTIONS", return_value={}) - with pytest.raises(RuntimeError) as context: - cli_run(["-p", of_id]) - assert repr(context.value) == repr( - RuntimeError(f"No virtualenv implementation for {PythonInfo.current_system(session_app_data, cache)}"), - ) +SYSTEM = PythonInfo.current_system() + + +@pytest.mark.parametrize( + "of_id", + ({sys.executable} if sys.executable != SYSTEM.executable else set()) | {SYSTEM.implementation}, +) +def test_failed_to_find_implementation(of_id, mocker): + mocker.patch("virtualenv.run.plugin.creators.CreatorSelector._OPTIONS", return_value={}) + with pytest.raises(RuntimeError) as context: + cli_run(["-p", of_id]) + assert repr(context.value) == repr(RuntimeError(f"No virtualenv implementation for {PythonInfo.current_system()}")) diff --git a/tests/unit/create/via_global_ref/test_build_c_ext.py b/tests/unit/create/via_global_ref/test_build_c_ext.py index b00a3eb47..5195ff856 100644 --- a/tests/unit/create/via_global_ref/test_build_c_ext.py +++ b/tests/unit/create/via_global_ref/test_build_c_ext.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging import os import shutil import subprocess @@ -9,12 +8,22 @@ import pytest -from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run -from virtualenv.run.plugin.creators import CreatorSelector -logger = logging.getLogger(__name__) +CURRENT = PythonInfo.current_system() +CREATOR_CLASSES = CURRENT.creators().key_to_class + + +def builtin_shows_marker_missing(): + builtin_classs = CREATOR_CLASSES.get("builtin") + if builtin_classs is None: + return False + host_include_marker = getattr(builtin_classs, "host_include_marker", None) + if host_include_marker is None: + return False + marker = host_include_marker(CURRENT) + return not marker.exists() @pytest.mark.slow @@ -23,63 +32,44 @@ strict=False, reason="did not manage to setup CI to run with VC 14.1 C++ compiler, but passes locally", ) -def test_can_build_c_extensions(tmp_path, coverage_env, session_app_data): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - current = PythonInfo.current_system(session_app_data, cache) - creator_classes = CreatorSelector.for_interpreter(current).key_to_class - - logger.warning("system_include: %s", current.system_include) - logger.warning("system_include exists: %s", Path(current.system_include).exists()) - - def builtin_shows_marker_missing(): - builtin_classs = creator_classes.get("builtin") - if builtin_classs is None: - return False - host_include_marker = getattr(builtin_classs, "host_include_marker", None) - if host_include_marker is None: - return False - marker = host_include_marker(current) - logger.warning("builtin marker: %s", marker) - logger.warning("builtin marker exists: %s", marker.exists()) - return not marker.exists() - - system_include = current.system_include - if not Path(system_include).exists() and not builtin_shows_marker_missing(): - pytest.skip("Building C-Extensions requires header files with host python") - - for creator in [i for i in creator_classes if i != "builtin"]: - env, greet = tmp_path / creator / "env", str(tmp_path / creator / "greet") - shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) - session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) - coverage_env() - setuptools_index_args = () - if current.version_info >= (3, 12): - # requires to be able to install setuptools as build dependency - setuptools_index_args = ( - "--find-links", - "https://pypi.org/simple/setuptools/", - ) +@pytest.mark.skipif( + not Path(CURRENT.system_include).exists() and not builtin_shows_marker_missing(), + reason="Building C-Extensions requires header files with host python", +) +@pytest.mark.parametrize("creator", [i for i in CREATOR_CLASSES if i != "builtin"]) +def test_can_build_c_extensions(creator, tmp_path, coverage_env): + env, greet = tmp_path / "env", str(tmp_path / "greet") + shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) + session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) + coverage_env() + setuptools_index_args = () + if CURRENT.version_info >= (3, 12): + # requires to be able to install setuptools as build dependency + setuptools_index_args = ( + "--find-links", + "https://pypi.org/simple/setuptools/", + ) - cmd = [ - str(session.creator.script("pip")), - "install", - "--no-index", - *setuptools_index_args, - "--no-deps", - "--disable-pip-version-check", - "-vvv", - greet, - ] - process = Popen(cmd) - process.communicate() - assert process.returncode == 0 + cmd = [ + str(session.creator.script("pip")), + "install", + "--no-index", + *setuptools_index_args, + "--no-deps", + "--disable-pip-version-check", + "-vvv", + greet, + ] + process = Popen(cmd) + process.communicate() + assert process.returncode == 0 - process = Popen( - [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], - universal_newlines=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) - out, _ = process.communicate() - assert process.returncode == 0 - assert out == "Hello World!\n" + process = Popen( + [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], + universal_newlines=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ) + out, _ = process.communicate() + assert process.returncode == 0 + assert out == "Hello World!\n" diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index fed641a98..15472daab 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -14,14 +14,13 @@ import pytest -from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew from virtualenv.discovery import cached_py_info from virtualenv.discovery.py_info import PythonInfo, VersionInfo from virtualenv.discovery.py_spec import PythonSpec from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink -CURRENT = PythonInfo.current_system(MockAppData(), MockCache()) +CURRENT = PythonInfo.current_system() def test_current_as_json(): @@ -33,21 +32,19 @@ def test_current_as_json(): assert parsed["free_threaded"] is f -def test_bad_exe_py_info_raise(tmp_path): +def test_bad_exe_py_info_raise(tmp_path, session_app_data): exe = str(tmp_path) - app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError) as context: - PythonInfo.from_exe(exe, app_data, cache) + PythonInfo.from_exe(exe, session_app_data) msg = str(context.value) assert "code" in msg assert exe in msg -def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys): +def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys, session_app_data): caplog.set_level(logging.NOTSET) exe = str(tmp_path) - app_data, cache = MockAppData(), MockCache() - result = PythonInfo.from_exe(exe, app_data, cache, raise_on_error=False) + result = PythonInfo.from_exe(exe, session_app_data, raise_on_error=False) assert result is None out, _ = capsys.readouterr() assert not out @@ -126,45 +123,41 @@ def test_satisfy_not_version(spec): assert matches is False -def test_py_info_cached_error(mocker, tmp_path): +def test_py_info_cached_error(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") - app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), app_data, cache) + PythonInfo.from_exe(str(tmp_path), session_app_data) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), app_data, cache) + PythonInfo.from_exe(str(tmp_path), session_app_data) assert spy.call_count == 1 @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink_error(mocker, tmp_path): +def test_py_info_cached_symlink_error(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") - app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), app_data, cache) + PythonInfo.from_exe(str(tmp_path), session_app_data) symlinked = tmp_path / "a" symlinked.symlink_to(tmp_path) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(symlinked), app_data, cache) + PythonInfo.from_exe(str(symlinked), session_app_data) assert spy.call_count == 2 -def test_py_info_cache_clear(mocker): +def test_py_info_cache_clear(mocker, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") - app_data, cache = MockAppData(), MockCache() - result = PythonInfo.from_exe(sys.executable, app_data, cache) + result = PythonInfo.from_exe(sys.executable, session_app_data) assert result is not None count = 1 if result.executable == sys.executable else 2 # at least two, one for the venv, one more for the host assert spy.call_count >= count - PythonInfo.clear_cache() - assert PythonInfo.from_exe(sys.executable, app_data, cache) is not None + PythonInfo.clear_cache(session_app_data) + assert PythonInfo.from_exe(sys.executable, session_app_data) is not None assert spy.call_count >= 2 * count -def test_py_info_cache_invalidation_on_py_info_change(mocker): +def test_py_info_cache_invalidation_on_py_info_change(mocker, session_app_data): # 1. Get a PythonInfo object for the current executable, this will cache it. - app_data, cache = MockAppData(), MockCache() - PythonInfo.from_exe(sys.executable, app_data, cache) + PythonInfo.from_exe(sys.executable, session_app_data) # 2. Spy on _run_subprocess spy = mocker.spy(cached_py_info, "_run_subprocess") @@ -182,7 +175,7 @@ def test_py_info_cache_invalidation_on_py_info_change(mocker): py_info_script.write_text(original_content + "\n# a comment", encoding="utf-8") # 6. Get the PythonInfo object again - info = PythonInfo.from_exe(sys.executable, app_data, cache) + info = PythonInfo.from_exe(sys.executable, session_app_data) # 7. Assert that _run_subprocess was called again native_difference = 1 if info.system_executable == info.executable else 0 @@ -204,10 +197,9 @@ def test_py_info_cache_invalidation_on_py_info_change(mocker): reason="symlink is not supported", ) @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink(mocker, tmp_path): +def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") - app_data, cache = MockAppData(), MockCache() - first_result = PythonInfo.from_exe(sys.executable, app_data, cache) + first_result = PythonInfo.from_exe(sys.executable, session_app_data) assert first_result is not None count = spy.call_count # at least two, one for the venv, one more for the host @@ -220,7 +212,7 @@ def test_py_info_cached_symlink(mocker, tmp_path): if pyvenv.exists(): (tmp_path / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") new_exe_str = str(new_exe) - second_result = PythonInfo.from_exe(new_exe_str, app_data, cache) + second_result = PythonInfo.from_exe(new_exe_str, session_app_data) assert second_result.executable == new_exe_str assert spy.call_count == count + 1 # no longer needed the host invocation, but the new symlink is must @@ -267,10 +259,10 @@ def test_system_executable_no_exact_match( # noqa: PLR0913 tmp_path, mocker, caplog, + session_app_data, ): """Here we should fallback to other compatible""" caplog.set_level(logging.DEBUG) - app_data, cache = MockAppData(), MockCache() def _make_py_info(of): base = copy.deepcopy(CURRENT) @@ -298,15 +290,15 @@ def _make_py_info(of): mocker.patch.object(target_py_info, "_find_possible_exe_names", return_value=names) mocker.patch.object(target_py_info, "_find_possible_folders", return_value=[str(tmp_path)]) - def func(exe, app_data, cache, raise_on_error=True, ignore_cache=False, resolve_to_host=True, env=None): # noqa: ARG001, PLR0913 - return discovered_with_path.get(exe) + def func(k, app_data, resolve_to_host, raise_on_error, env): # noqa: ARG001 + return discovered_with_path[k] - mocker.patch.object(PythonInfo, "from_exe", side_effect=func) + mocker.patch.object(target_py_info, "from_exe", side_effect=func) target_py_info.real_prefix = str(tmp_path) target_py_info.system_executable = None target_py_info.executable = str(tmp_path) - mapped = target_py_info._resolve_to_system(app_data, target_py_info, cache) # noqa: SLF001 + mapped = target_py_info._resolve_to_system(session_app_data, target_py_info) # noqa: SLF001 assert mapped.system_executable == CURRENT.system_executable found = discovered_with_path[mapped.base_executable] assert found is selected @@ -333,8 +325,7 @@ def test_py_info_ignores_distutils_config(monkeypatch, tmp_path): """ (tmp_path / "setup.cfg").write_text(dedent(raw), encoding="utf-8") monkeypatch.chdir(tmp_path) - app_data, cache = MockAppData(), MockCache() - py_info = PythonInfo.from_exe(sys.executable, app_data, cache) + py_info = PythonInfo.from_exe(sys.executable) distutils = py_info.distutils_install for key, value in distutils.items(): assert not value.startswith(str(tmp_path)), f"{key}={value}" @@ -371,11 +362,10 @@ def test_py_info_setuptools(): @pytest.mark.usefixtures("_skip_if_test_in_system") -def test_py_info_to_system_raises(mocker, caplog): +def test_py_info_to_system_raises(session_app_data, mocker, caplog): caplog.set_level(logging.DEBUG) mocker.patch.object(PythonInfo, "_find_possible_folders", return_value=[]) - app_data, cache = MockAppData(), MockCache() - result = PythonInfo.from_exe(sys.executable, app_data=app_data, cache=cache, raise_on_error=False) + result = PythonInfo.from_exe(sys.executable, app_data=session_app_data, raise_on_error=False) assert result is None log = caplog.records[-1] assert log.levelno == logging.INFO diff --git a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py index ec3def025..90894a59c 100644 --- a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py +++ b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py @@ -6,17 +6,15 @@ import pytest -from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.py_info import EXTENSIONS, PythonInfo from virtualenv.info import IS_WIN, fs_is_case_sensitive, fs_supports_symlink -CURRENT = PythonInfo.current(MockAppData(), MockCache()) +CURRENT = PythonInfo.current() -def test_discover_empty_folder(tmp_path): - app_data, cache = MockAppData(), MockCache() +def test_discover_empty_folder(tmp_path, session_app_data): with pytest.raises(RuntimeError): - CURRENT.discover_exe(app_data, prefix=str(tmp_path), cache=cache) + CURRENT.discover_exe(session_app_data, prefix=str(tmp_path)) BASE = (CURRENT.install_path("scripts"), ".") @@ -28,9 +26,8 @@ def test_discover_empty_folder(tmp_path): @pytest.mark.parametrize("arch", [CURRENT.architecture, ""]) @pytest.mark.parametrize("version", [".".join(str(i) for i in CURRENT.version_info[0:i]) for i in range(3, 0, -1)]) @pytest.mark.parametrize("impl", [CURRENT.implementation, "python"]) -def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog): # noqa: PLR0913 +def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, session_app_data): # noqa: PLR0913 caplog.set_level(logging.DEBUG) - app_data, cache = MockAppData(), MockCache() folder = tmp_path / into folder.mkdir(parents=True, exist_ok=True) name = f"{impl}{version}{'t' if CURRENT.free_threaded else ''}" @@ -43,7 +40,7 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog): # no if pyvenv.exists(): (folder / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") inside_folder = str(tmp_path) - base = CURRENT.discover_exe(app_data, inside_folder, cache=cache) + base = CURRENT.discover_exe(session_app_data, inside_folder) found = base.executable dest_str = str(dest) if not fs_is_case_sensitive(): @@ -56,4 +53,4 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog): # no dest.rename(dest.parent / (dest.name + "-1")) CURRENT._cache_exe_discovery.clear() # noqa: SLF001 with pytest.raises(RuntimeError): - CURRENT.discover_exe(app_data, inside_folder, cache=cache) + CURRENT.discover_exe(session_app_data, inside_folder) diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index 68de1da56..e6c78e2e3 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -11,7 +11,6 @@ import pytest -from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.builtin import Builtin, LazyPathDump, get_interpreter, get_paths from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_WIN, fs_supports_symlink @@ -20,10 +19,9 @@ @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") @pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) @pytest.mark.parametrize("specificity", ["more", "less", "none"]) -def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog): +def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog, session_app_data): # noqa: PLR0913 caplog.set_level(logging.DEBUG) - app_data, cache = MockAppData(), MockCache() - current = PythonInfo.current_system(app_data, cache) + current = PythonInfo.current_system(session_app_data) name = "somethingVeryCryptic" threaded = "t" if current.free_threaded else "" if case == "lower": @@ -53,39 +51,36 @@ def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog): (target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes()) new_path = os.pathsep.join([str(target), *os.environ.get("PATH", "").split(os.pathsep)]) monkeypatch.setenv("PATH", new_path) - interpreter = get_interpreter(core, [], app_data, cache) + interpreter = get_interpreter(core, []) assert interpreter is not None def test_discovery_via_path_not_found(tmp_path, monkeypatch): monkeypatch.setenv("PATH", str(tmp_path)) - app_data, cache = MockAppData(), MockCache() - interpreter = get_interpreter(uuid4().hex, [], app_data, cache) + interpreter = get_interpreter(uuid4().hex, []) assert interpreter is None def test_discovery_via_path_in_nonbrowseable_directory(tmp_path, monkeypatch): bad_perm = tmp_path / "bad_perm" bad_perm.mkdir(mode=0o000) - app_data, cache = MockAppData(), MockCache() # path entry is unreadable monkeypatch.setenv("PATH", str(bad_perm)) - interpreter = get_interpreter(uuid4().hex, [], app_data, cache) + interpreter = get_interpreter(uuid4().hex, []) assert interpreter is None # path entry parent is unreadable monkeypatch.setenv("PATH", str(bad_perm / "bin")) - interpreter = get_interpreter(uuid4().hex, [], app_data, cache) + interpreter = get_interpreter(uuid4().hex, []) assert interpreter is None -def test_relative_path(monkeypatch): - app_data, cache = MockAppData(), MockCache() - sys_executable = Path(PythonInfo.current_system(app_data=app_data, cache=cache).system_executable) +def test_relative_path(session_app_data, monkeypatch): + sys_executable = Path(PythonInfo.current_system(app_data=session_app_data).system_executable) cwd = sys_executable.parents[1] monkeypatch.chdir(str(cwd)) relative = str(sys_executable.relative_to(cwd)) - result = get_interpreter(relative, [], app_data, cache) + result = get_interpreter(relative, [], session_app_data) assert result is not None @@ -100,14 +95,13 @@ def test_uv_python(monkeypatch, tmp_path_factory, mocker): with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: m.setenv("UV_PYTHON_INSTALL_DIR", str(uv_python_install_dir)) - app_data, cache = MockAppData(), MockCache() - get_interpreter("python", [], app_data, cache) + get_interpreter("python", []) mock_from_exe.assert_not_called() bin_path = uv_python_install_dir.joinpath("some-py-impl", "bin") bin_path.mkdir(parents=True) bin_path.joinpath("python").touch() - get_interpreter("python", [], app_data, cache) + get_interpreter("python", []) mock_from_exe.assert_called_once() assert mock_from_exe.call_args[0][0] == str(bin_path / "python") @@ -126,14 +120,13 @@ def test_uv_python(monkeypatch, tmp_path_factory, mocker): with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: m.setenv("XDG_DATA_HOME", str(xdg_data_home)) - app_data, cache = MockAppData(), MockCache() - get_interpreter("python", [], app_data, cache) + get_interpreter("python", []) mock_from_exe.assert_not_called() bin_path = xdg_data_home.joinpath("uv", "python", "some-py-impl", "bin") bin_path.mkdir(parents=True) bin_path.joinpath("python").touch() - get_interpreter("python", [], app_data, cache) + get_interpreter("python", []) mock_from_exe.assert_called_once() assert mock_from_exe.call_args[0][0] == str(bin_path / "python") @@ -142,24 +135,21 @@ def test_uv_python(monkeypatch, tmp_path_factory, mocker): with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: m.setattr("virtualenv.discovery.builtin.user_data_path", lambda x: user_data_path / x) - app_data, cache = MockAppData(), MockCache() - get_interpreter("python", [], app_data, cache) + get_interpreter("python", []) mock_from_exe.assert_not_called() bin_path = user_data_path.joinpath("uv", "python", "some-py-impl", "bin") bin_path.mkdir(parents=True) bin_path.joinpath("python").touch() - get_interpreter("python", [], app_data, cache) + get_interpreter("python", []) mock_from_exe.assert_called_once() assert mock_from_exe.call_args[0][0] == str(bin_path / "python") -def test_discovery_fallback_fail(caplog): +def test_discovery_fallback_fail(session_app_data, caplog): caplog.set_level(logging.DEBUG) - app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ), - cache, + Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ), ) result = builtin.run() @@ -168,12 +158,10 @@ def test_discovery_fallback_fail(caplog): assert "accepted" not in caplog.text -def test_discovery_fallback_ok(caplog): +def test_discovery_fallback_ok(session_app_data, caplog): caplog.set_level(logging.DEBUG) - app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ), - cache, + Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ), ) result = builtin.run() @@ -192,12 +180,10 @@ def mock_get_interpreter(mocker): @pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocker, monkeypatch): +def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocker, monkeypatch, session_app_data): monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") - app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=app_data, try_first_with=[], python=["python_from_env_var"], env=os.environ), - cache, + Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_env_var"], env=os.environ), ) result = builtin.run() @@ -206,17 +192,17 @@ def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocke @pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_second_python_specified_when_more_than_one_is_specified_and_env_var_is_specified(mocker, monkeypatch): +def test_returns_second_python_specified_when_more_than_one_is_specified_and_env_var_is_specified( + mocker, monkeypatch, session_app_data +): monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") - app_data, cache = MockAppData(), MockCache() builtin = Builtin( Namespace( - app_data=app_data, + app_data=session_app_data, try_first_with=[], python=["python_from_env_var", "python_from_cli"], env=os.environ, ), - cache, ) result = builtin.run() @@ -224,19 +210,7 @@ def test_returns_second_python_specified_when_more_than_one_is_specified_and_env assert result == mocker.sentinel.python_from_cli -def test_get_interpreter_no_cache_no_app_data(): - """Test that get_interpreter can be called without cache and app_data.""" - # A call to a valid interpreter should succeed and return a PythonInfo object. - interpreter = get_interpreter(sys.executable, []) - assert interpreter is not None - assert Path(interpreter.executable).is_file() - - # A call to an invalid interpreter should not fail and should return None. - interpreter = get_interpreter("a-python-that-does-not-exist", []) - assert interpreter is None - - -def test_discovery_absolute_path_with_try_first(tmp_path): +def test_discovery_absolute_path_with_try_first(tmp_path, session_app_data): good_env = tmp_path / "good" bad_env = tmp_path / "bad" @@ -252,12 +226,10 @@ def test_discovery_absolute_path_with_try_first(tmp_path): # The spec is an absolute path, this should be a hard requirement. # The --try-first-with option should be rejected as it does not match the spec. - app_data, cache = MockAppData(), MockCache() interpreter = get_interpreter( str(good_exe), try_first_with=[str(bad_exe)], - app_data=app_data, - cache=cache, + app_data=session_app_data, ) assert interpreter is not None @@ -268,8 +240,7 @@ def test_discovery_via_path_with_file(tmp_path, monkeypatch): a_file = tmp_path / "a_file" a_file.touch() monkeypatch.setenv("PATH", str(a_file)) - app_data, cache = MockAppData(), MockCache() - interpreter = get_interpreter(uuid4().hex, [], app_data, cache) + interpreter = get_interpreter(uuid4().hex, []) assert interpreter is None @@ -349,12 +320,10 @@ def test_lazy_path_dump_debug(monkeypatch, tmp_path): @pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_first_python_specified_when_no_env_var_is_specified(mocker, monkeypatch): +def test_returns_first_python_specified_when_no_env_var_is_specified(mocker, monkeypatch, session_app_data): monkeypatch.delenv("VIRTUALENV_PYTHON", raising=False) - app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=app_data, try_first_with=[], python=["python_from_cli"], env=os.environ), - cache, + Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_cli"], env=os.environ), ) result = builtin.run() diff --git a/tests/unit/discovery/util.py b/tests/unit/discovery/util.py deleted file mode 100644 index 7908c7df8..000000000 --- a/tests/unit/discovery/util.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, ContextManager - -if TYPE_CHECKING: - from pathlib import Path - - -class MockAppData: - def __init__(self, readonly: bool = False) -> None: - self.readonly = readonly - self._py_info_clear_called = 0 - self._py_info_map: dict[Path, Any] = {} - - def py_info(self, path: Path) -> Any: - return self._py_info_map.get(path) - - def py_info_clear(self) -> None: - self._py_info_clear_called += 1 - - @contextmanager - def ensure_extracted(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: # noqa: ARG002 - yield path - - @contextmanager - def extract(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: # noqa: ARG002 - yield path - - def close(self) -> None: - pass - - -class MockCache: - def __init__(self) -> None: - self._cache: dict[Path, Any] = {} - self._clear_called = 0 - - def get(self, path: Path) -> Any: - return self._cache.get(path) - - def set(self, path: Path, data: Any) -> None: - self._cache[path] = data - - def remove(self, path: Path) -> None: - if path in self._cache: - del self._cache[path] - - def clear(self) -> None: - self._clear_called += 1 - self._cache.clear() diff --git a/tests/unit/discovery/windows/test_windows.py b/tests/unit/discovery/windows/test_windows.py index 98b849f57..594a1302f 100644 --- a/tests/unit/discovery/windows/test_windows.py +++ b/tests/unit/discovery/windows/test_windows.py @@ -4,7 +4,6 @@ import pytest -from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.py_spec import PythonSpec @@ -37,5 +36,5 @@ def test_propose_interpreters(string_spec, expected_exe): from virtualenv.discovery.windows import propose_interpreters # noqa: PLC0415 spec = PythonSpec.from_string_spec(string_spec) - interpreter = next(propose_interpreters(spec, MockAppData(), MockCache(), env=None)) + interpreter = next(propose_interpreters(spec=spec, cache_dir=None, env=None)) assert interpreter.executable == expected_exe diff --git a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py index b37b7cdbb..4076573e1 100644 --- a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py +++ b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py @@ -10,7 +10,6 @@ import pytest -from virtualenv.cache import FileCache from virtualenv.discovery import cached_py_info from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink @@ -26,16 +25,8 @@ @pytest.mark.slow @pytest.mark.parametrize("copies", [False, True] if fs_supports_symlink() else [True]) -def test_seed_link_via_app_data( # noqa: PLR0913, PLR0915 - tmp_path, - coverage_env, - current_fastest, - copies, - for_py_version, - session_app_data, -): - cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) - current = PythonInfo.current_system(session_app_data, cache) +def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies, for_py_version): # noqa: PLR0915 + current = PythonInfo.current_system() bundle_ver = BUNDLE_SUPPORT[current.version_release_str] create_cmd = [ str(tmp_path / "en v"), # space in the name to ensure generated scripts work when path has space diff --git a/tests/unit/test_file_limit.py b/tests/unit/test_file_limit.py index 0c66561f8..df2a281e8 100644 --- a/tests/unit/test_file_limit.py +++ b/tests/unit/test_file_limit.py @@ -1,12 +1,10 @@ from __future__ import annotations -import errno import os import sys import pytest -from virtualenv.info import IMPLEMENTATION from virtualenv.run import cli_run @@ -18,34 +16,24 @@ def test_too_many_open_files(tmp_path): import resource # noqa: PLC0415 soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft_limit > 1024: + pytest.skip("soft limit for open files is too high to reliably trigger the error") # Lower the soft limit to a small number to trigger the error try: resource.setrlimit(resource.RLIMIT_NOFILE, (32, hard_limit)) except ValueError: pytest.skip("could not lower the soft limit for open files") - except AttributeError as exc: # pypy, graalpy - if "module 'resource' has no attribute 'setrlimit'" in str(exc): - pytest.skip(f"{IMPLEMENTATION} does not support resource.setrlimit") # Keep some file descriptors open to make it easier to trigger the error fds = [] try: - # JIT implementations use more file descriptors up front so we can run out early - try: - fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) - except OSError as jit_exceptions: # pypy, graalpy - assert jit_exceptions.errno == errno.EMFILE # noqa: PT017 - assert "Too many open files" in str(jit_exceptions) # noqa: PT017 - - expected_exceptions = SystemExit, OSError, RuntimeError - with pytest.raises(expected_exceptions) as too_many_open_files_exc: + fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) + + with pytest.raises(SystemExit) as excinfo: cli_run([str(tmp_path / "venv")]) - if isinstance(too_many_open_files_exc, SystemExit): - assert too_many_open_files_exc.code != 0 - else: - assert "Too many open files" in str(too_many_open_files_exc.value) + assert excinfo.value.code != 0 finally: for fd in fds: From 53872861ccf957ef7df2ba2020fba759f7b9c914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 10 Oct 2025 09:52:28 -0700 Subject: [PATCH 15/17] release 20.35.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- docs/changelog.rst | 7 +++++++ docs/changelog/2978.bugfix.rst | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) delete mode 100644 docs/changelog/2978.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 638e43f46..3ec2e6449 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,13 @@ Release History .. towncrier release notes start +v20.35.2 (2025-10-10) +--------------------- + +Bugfixes - 20.35.2 +~~~~~~~~~~~~~~~~~~ +- Revert out changes related to the extraction of the discovery module - by :user:`gaborbernat`. (:issue:`2978`) + v20.35.1 (2025-10-09) --------------------- diff --git a/docs/changelog/2978.bugfix.rst b/docs/changelog/2978.bugfix.rst deleted file mode 100644 index 013023fe3..000000000 --- a/docs/changelog/2978.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Revert out changes related to the extraction of the discovery module - by :user:`gaborbernat`. From a07135f34ad8f30c9921f05dc913ccd83e06679d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 10 Oct 2025 11:32:38 -0700 Subject: [PATCH 16/17] test_too_many_open_files fails (#2979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Emre Şafak <3928300+esafak@users.noreply.github.com> --- docs/changelog/2935.bugfix.rst | 1 + tests/unit/test_file_limit.py | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/2935.bugfix.rst diff --git a/docs/changelog/2935.bugfix.rst b/docs/changelog/2935.bugfix.rst new file mode 100644 index 000000000..758553f8a --- /dev/null +++ b/docs/changelog/2935.bugfix.rst @@ -0,0 +1 @@ +Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak` diff --git a/tests/unit/test_file_limit.py b/tests/unit/test_file_limit.py index df2a281e8..0c66561f8 100644 --- a/tests/unit/test_file_limit.py +++ b/tests/unit/test_file_limit.py @@ -1,10 +1,12 @@ from __future__ import annotations +import errno import os import sys import pytest +from virtualenv.info import IMPLEMENTATION from virtualenv.run import cli_run @@ -16,24 +18,34 @@ def test_too_many_open_files(tmp_path): import resource # noqa: PLC0415 soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - if soft_limit > 1024: - pytest.skip("soft limit for open files is too high to reliably trigger the error") # Lower the soft limit to a small number to trigger the error try: resource.setrlimit(resource.RLIMIT_NOFILE, (32, hard_limit)) except ValueError: pytest.skip("could not lower the soft limit for open files") + except AttributeError as exc: # pypy, graalpy + if "module 'resource' has no attribute 'setrlimit'" in str(exc): + pytest.skip(f"{IMPLEMENTATION} does not support resource.setrlimit") # Keep some file descriptors open to make it easier to trigger the error fds = [] try: - fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) - - with pytest.raises(SystemExit) as excinfo: + # JIT implementations use more file descriptors up front so we can run out early + try: + fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) + except OSError as jit_exceptions: # pypy, graalpy + assert jit_exceptions.errno == errno.EMFILE # noqa: PT017 + assert "Too many open files" in str(jit_exceptions) # noqa: PT017 + + expected_exceptions = SystemExit, OSError, RuntimeError + with pytest.raises(expected_exceptions) as too_many_open_files_exc: cli_run([str(tmp_path / "venv")]) - assert excinfo.value.code != 0 + if isinstance(too_many_open_files_exc, SystemExit): + assert too_many_open_files_exc.code != 0 + else: + assert "Too many open files" in str(too_many_open_files_exc.value) finally: for fd in fds: From 8a1160e2bab063f6d6ab6295052cece8dd805cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 10 Oct 2025 14:22:45 -0700 Subject: [PATCH 17/17] release 20.35.3 --- docs/changelog.rst | 7 +++++++ docs/changelog/2935.bugfix.rst | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) delete mode 100644 docs/changelog/2935.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 3ec2e6449..d0682c90e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,13 @@ Release History .. towncrier release notes start +v20.35.3 (2025-10-10) +--------------------- + +Bugfixes - 20.35.3 +~~~~~~~~~~~~~~~~~~ +- Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak` (:issue:`2935`) + v20.35.2 (2025-10-10) --------------------- diff --git a/docs/changelog/2935.bugfix.rst b/docs/changelog/2935.bugfix.rst deleted file mode 100644 index 758553f8a..000000000 --- a/docs/changelog/2935.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak`