From 2ee7aa24245cc7b76a1fa3fb33970d8f3ececf55 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Mon, 16 Nov 2026 14:28:50 -0700 Subject: [PATCH 1/4] test: The red check and the push gate give the runner only files the tests commit leaves in place The tests come first and fail until the implementation. The unit `di-repo-tidy/1` failed its red check three times with "nothing failed at the tests commit". Unit: red-check-skips-deleted-test-files/1 --- .../pipeline/test_red_check_runnable_files.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/pipeline/test_red_check_runnable_files.py b/tests/pipeline/test_red_check_runnable_files.py index 6e9798223..68c75abee 100644 --- a/tests/pipeline/test_red_check_runnable_files.py +++ b/tests/pipeline/test_red_check_runnable_files.py @@ -108,6 +108,62 @@ def test_a_tests_commit_with_only_a_data_file_builds_no_red_command(repo: Path) assert red_problems(repo, git(repo, "rev-parse", "HEAD~1"), profile=profile.toolchain) == [] +def test_a_deleted_test_file_is_not_named_in_the_narrowed_run_or_the_run_of_the_files_whole( + repo: Path, +) -> None: + git(repo, "rm", "-q", "tests/test_old.py") + commit(repo, "test: moves one", {"tests/test_new.py": FAILING}) + profile = Recording(PYTHON) + + red_problems(repo, git(repo, "rev-parse", "HEAD"), profile=profile.toolchain) + + (command,) = profile.commands + assert "test_old" not in command, "neither run names the deleted file" + words = shlex.split(command) + assert "tests/test_new.py::test_new" in words, "the narrowed run" + assert "tests/test_new.py" in words, "the run of the files whole" + assert profile.files == [["tests/test_new.py"]] + + +def test_the_push_gate_does_not_give_the_runner_a_deleted_test_file(repo: Path) -> None: + git(repo, "rm", "-q", "tests/test_old.py") + commit(repo, "test: moves one", {"tests/test_new.py": FAILING}) + commit(repo, "feat: implements it", {"src/new.py": "x = 1\n"}) + profile = Recording(PYTHON) + + problems = check_branch(repo, "main", profile=profile.toolchain) + + assert problems == [] + assert profile.files == [["tests/test_new.py"]] + + +def test_a_renamed_test_file_runs_under_its_new_path_only(repo: Path) -> None: + git(repo, "mv", "tests/test_old.py", "tests/test_renamed.py") + commit(repo, "test: renames one", {"tests/test_renamed.py": FAILING}) + profile = Recording(PYTHON) + + red_problems(repo, git(repo, "rev-parse", "HEAD"), profile=profile.toolchain) + + assert profile.files == [["tests/test_renamed.py"]] + assert "test_old" not in profile.commands[0] + + +def test_a_tests_commit_that_only_deletes_a_test_file_is_a_tests_commit_with_no_red_command( + repo: Path, +) -> None: + git(repo, "rm", "-q", "tests/test_old.py") + commit(repo, "test: removes one", {}) + commit(repo, "feat: code", {"src/thing.py": "x = 1\n"}) + profile = Recording(PYTHON) + + problems = check_branch(repo, "main", profile=profile.toolchain) + + assert problems == [], "the deletion counts as the tests commit that precedes the code" + assert profile.commands == [] + assert red_problems(repo, git(repo, "rev-parse", "HEAD~1"), profile=profile.toolchain) == [] + assert profile.commands == [] + + @pytest.mark.parametrize( ("path", "runnable"), [ From 84123f4a80522c51233ff291e6a092fd9c754523 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Mon, 16 Nov 2026 14:30:45 -0700 Subject: [PATCH 2/4] feat: The red check and the push gate give the runner only files the tests commit leaves in place With this in, the tests of the previous commit pass. The unit `di-repo-tidy/1` failed its red check three times with "nothing failed at the tests commit". Unit: red-check-skips-deleted-test-files/1 --- CHANGELOG.md | 5 +++++ docs/toolchain-profiles.md | 2 +- src/agent_build_kit/pipeline/gate.py | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a62202ce..4c83a75c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- The red check and the push gate no longer give the test runner a test file the tests commit + deletes or renames away. Before, the runner was named a path that no longer existed and + answered with a usage error, so the check read an empty report as "nothing failed" and + rejected the unit; a commit that only deletes test files now builds no red command. + - The rerun of a failed test now collects a test of the `integration` kind that a repository's default settings deselect. Before, the rerun carried no marker expression, so such a test was deselected (pytest exit 5), never ran again, and a flake in it failed the unit; the rerun now diff --git a/docs/toolchain-profiles.md b/docs/toolchain-profiles.md index 4d6ff2f05..55226d388 100644 --- a/docs/toolchain-profiles.md +++ b/docs/toolchain-profiles.md @@ -43,7 +43,7 @@ Two ship: `python-uv` and `node-npm`, both implemented. | `e2e_commands(repo, marker)` | e2e-test without a dev stack | every member's live-stack tests | | `acceptance_commands(checkout, paths, marker, exclude_marker, root_extras)` | post-merge verify | the live tests among a change's paths | | `clean_command(*, types=False)` | the push gate | lint and format at the tests commit, types skipped; with `types=True` the type check runs too, as the gate does for a stubbed build | -| `red_command(files, ids=None)` | the push gate | run these test files at the tests commit (only files `runs_path` accepts: a data file under a test folder is part of the tests commit but is not given to the runner); `ids` (`path::name`) are the tests the commit added or changed, which a runner that selects by id runs in place of the files | +| `red_command(files, ids=None)` | the push gate | run these test files at the tests commit (only files `runs_path` accepts: a data file under a test folder is part of the tests commit but is not given to the runner, and neither is a file the commit deletes or the old path of a rename); `ids` (`path::name`) are the tests the commit added or changed, which a runner that selects by id runs in place of the files | | `interpret_red(output, exit_code, *, stubbed=False)` | the push gate | was that run honestly red, and why not; with `stubbed=True` the stubbed red reading: the test must fail by an assertion or a "not implemented" error, and a failure to import or a missing attribute is refused | | `fix_command(files)` | the build, before the tests, implementation and checks-loop `fix:` commits | optional: the argv that applies the profile's formatters and auto-fixers to `files` (the unit's own changed and untracked files of one project, relative to its directory), or `None` when none of them is one it fixes. Exit status is ignored; the runner judges by the files. python-uv: `uv run pre-commit run --files `; node-npm: `npx oxfmt` then `npx oxlint --fix` over the source files, never `tsc`. A profile without it gets no fixer pass; `repos..tests.fix_command` replaces it (`[]` turns it off). To add `fix_command` to a profile, return the argv for the files it fixes and `None` for the rest | | `supports_stubs` | the build, the gate | optional boolean: whether the profile can write stubs that pass its static checks. Read tolerantly (`profiles.base.supports_stubs`): a profile without it, or with anything but true, does not support stubs, so `builders.stubbed_tdd_commits` has no effect on its repos | diff --git a/src/agent_build_kit/pipeline/gate.py b/src/agent_build_kit/pipeline/gate.py index 63e97559f..4866d1797 100644 --- a/src/agent_build_kit/pipeline/gate.py +++ b/src/agent_build_kit/pipeline/gate.py @@ -66,6 +66,15 @@ def _test_files_in(repo: Path, sha: str, is_test: Callable[[str], bool]) -> list return [path for path, kind in classify_paths(files, is_test).items() if kind == "test"] +def _left_in_place(repo: Path, sha: str, files: list[str]) -> list[str]: + """The `files` the commit `sha` holds: a file it deletes, or the old path of a rename, is + still part of the commit but is not one the runner can be given.""" + if not files: + return [] + held = set(git(repo, "ls-tree", "-r", "--name-only", sha, "--", *files).stdout.split("\n")) + return [file for file in files if file in held] + + def _touched_in(repo: Path, sha: str) -> list[tuple[str, str]]: """The tests the commit adds or changes against its parent; none when they cannot be listed (a root commit has no parent), so its files run whole.""" @@ -198,7 +207,7 @@ def is_test(path: str) -> bool: for owner, owned in by_project(projects, _test_files_in(repo, sha, is_test)): toolchain = profiles.for_project(owner, profile) prefix = owner.prefix if owner else "" - command = _red_command(toolchain, owned, touched, repo, prefix) + command = _red_command(toolchain, _left_in_place(repo, sha, owned), touched, repo, prefix) if command is None: continue @@ -306,7 +315,11 @@ def clean_in( commands: list[str | Callable[[Path], str]] = [clean_in] red_line = ( - None if sha in no_red else _red_command(toolchain, owned, touched, repo, prefix) + None + if sha in no_red + else _red_command( + toolchain, _left_in_place(repo, sha, owned), touched, repo, prefix + ) ) if red_line is not None: commands.append(red_line) From 9d6050f3d8d6705578f2e9dbe63174fa1040ab8e Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Mon, 16 Nov 2026 15:34:21 -0700 Subject: [PATCH 3/4] fix: The red check and the push gate give the runner only files the tests commit leaves in place (checks, round 1) Unit: red-check-skips-deleted-test-files/1 --- src/agent_build_kit/pipeline/gate.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/agent_build_kit/pipeline/gate.py b/src/agent_build_kit/pipeline/gate.py index 4866d1797..161b3ee52 100644 --- a/src/agent_build_kit/pipeline/gate.py +++ b/src/agent_build_kit/pipeline/gate.py @@ -66,13 +66,15 @@ def _test_files_in(repo: Path, sha: str, is_test: Callable[[str], bool]) -> list return [path for path, kind in classify_paths(files, is_test).items() if kind == "test"] -def _left_in_place(repo: Path, sha: str, files: list[str]) -> list[str]: - """The `files` the commit `sha` holds: a file it deletes, or the old path of a rename, is - still part of the commit but is not one the runner can be given.""" +def _left_in_place(repo: Path, sha: str, files: list[str], prefix: str) -> list[str]: + """The `files` (relative to the project at `prefix`) the commit `sha` holds: a file it + deletes, or the old path of a rename, is still part of the commit but is not one the + runner can be given.""" if not files: return [] - held = set(git(repo, "ls-tree", "-r", "--name-only", sha, "--", *files).stdout.split("\n")) - return [file for file in files if file in held] + paths = [prefix + file for file in files] + held = set(git(repo, "ls-tree", "-r", "--name-only", sha, "--", *paths).stdout.split("\n")) + return [file for file, path in zip(files, paths, strict=True) if path in held] def _touched_in(repo: Path, sha: str) -> list[tuple[str, str]]: @@ -207,7 +209,8 @@ def is_test(path: str) -> bool: for owner, owned in by_project(projects, _test_files_in(repo, sha, is_test)): toolchain = profiles.for_project(owner, profile) prefix = owner.prefix if owner else "" - command = _red_command(toolchain, _left_in_place(repo, sha, owned), touched, repo, prefix) + left = _left_in_place(repo, sha, owned, prefix) + command = _red_command(toolchain, left, touched, repo, prefix) if command is None: continue @@ -318,7 +321,7 @@ def clean_in( None if sha in no_red else _red_command( - toolchain, _left_in_place(repo, sha, owned), touched, repo, prefix + toolchain, _left_in_place(repo, sha, owned, prefix), touched, repo, prefix ) ) if red_line is not None: From 85b158cd25c41f859a7be4971fe66edb1e0f7a8f Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Mon, 16 Nov 2026 16:33:54 -0700 Subject: [PATCH 4/4] fix: The red check and the push gate give the runner only files the tests commit leaves in place (review round 1) Unit: red-check-skips-deleted-test-files/1 --- src/agent_build_kit/pipeline/gate.py | 23 +++++----- .../pipeline/test_red_check_runnable_files.py | 42 +++++++++++++++++++ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/agent_build_kit/pipeline/gate.py b/src/agent_build_kit/pipeline/gate.py index 161b3ee52..7a166bf17 100644 --- a/src/agent_build_kit/pipeline/gate.py +++ b/src/agent_build_kit/pipeline/gate.py @@ -73,7 +73,8 @@ def _left_in_place(repo: Path, sha: str, files: list[str], prefix: str) -> list[ if not files: return [] paths = [prefix + file for file in files] - held = set(git(repo, "ls-tree", "-r", "--name-only", sha, "--", *paths).stdout.split("\n")) + listing = git(repo, "ls-tree", "-r", "--name-only", "-z", sha, "--", *paths).stdout + held = set(listing.split("\0")) return [file for file, path in zip(files, paths, strict=True) if path in held] @@ -92,17 +93,20 @@ def _red_command( touched: list[tuple[str, str]], repo: Path, prefix: str, + sha: str, ) -> str | None: - """The red command for `owned` (the project's test files at a tests commit), narrowed to - the tests the commit added or changed (`touched`, from the checkout's root). + """The red command for `owned` (the project's test files at the tests commit `sha`), + narrowed to the tests the commit added or changed (`touched`, from the checkout's root). - Only the files the toolchain can run are given to the runner (a data file under a test - folder is part of the commit but is not one); with none, there is no command. + Only the files the toolchain can run and the commit leaves in place are given to the + runner (a data file under a test folder is part of the commit but is not runnable; a + deleted file or the old path of a rename is part of it but is gone); with none, there is + no command. A file with no selected test (a helper edit) stays whole beside the selected ids. With no test selected in any file the command is the whole-file run. """ - owned = [file for file in owned if toolchain.runs_path(file)] + owned = _left_in_place(repo, sha, [f for f in owned if toolchain.runs_path(f)], prefix) if not owned: return None ids = ids_for(touched, repo, repo / prefix, toolchain) @@ -209,8 +213,7 @@ def is_test(path: str) -> bool: for owner, owned in by_project(projects, _test_files_in(repo, sha, is_test)): toolchain = profiles.for_project(owner, profile) prefix = owner.prefix if owner else "" - left = _left_in_place(repo, sha, owned, prefix) - command = _red_command(toolchain, left, touched, repo, prefix) + command = _red_command(toolchain, owned, touched, repo, prefix, sha) if command is None: continue @@ -320,9 +323,7 @@ def clean_in( red_line = ( None if sha in no_red - else _red_command( - toolchain, _left_in_place(repo, sha, owned, prefix), touched, repo, prefix - ) + else _red_command(toolchain, owned, touched, repo, prefix, sha) ) if red_line is not None: commands.append(red_line) diff --git a/tests/pipeline/test_red_check_runnable_files.py b/tests/pipeline/test_red_check_runnable_files.py index 68c75abee..038b83bf7 100644 --- a/tests/pipeline/test_red_check_runnable_files.py +++ b/tests/pipeline/test_red_check_runnable_files.py @@ -137,6 +137,48 @@ def test_the_push_gate_does_not_give_the_runner_a_deleted_test_file(repo: Path) assert profile.files == [["tests/test_new.py"]] +NODE_FAILING = "test('x', () => { throw new Error('no') })\n" +WEB = [ProjectConfig(path="web", profile="node-npm")] + + +@pytest.fixture +def web_repo(repo: Path) -> Path: + """`repo` with a node project's test file, `web/src/old.test.ts`, already on main.""" + git(repo, "checkout", "-q", "main") + commit(repo, "web", {"web/src/old.test.ts": NODE_FAILING}) + git(repo, "checkout", "-q", "spec/change/1-unit") + git(repo, "rebase", "-q", "main") + return repo + + +def test_a_deleted_test_file_of_a_project_below_the_root_is_left_out_of_the_red_run( + web_repo: Path, +) -> None: + git(web_repo, "rm", "-q", "web/src/old.test.ts") + commit(web_repo, "test: moves one", {"web/src/new.test.ts": NODE_FAILING}) + profile = Recording(NODE) + + with profiles.registered(profile.toolchain): + red_problems(web_repo, git(web_repo, "rev-parse", "HEAD"), profile=PYTHON, projects=WEB) + + assert profile.files == [["src/new.test.ts"]] + + +def test_the_push_gate_leaves_a_deleted_test_file_of_a_project_below_the_root_out( + web_repo: Path, +) -> None: + git(web_repo, "rm", "-q", "web/src/old.test.ts") + commit(web_repo, "test: moves one", {"web/src/new.test.ts": NODE_FAILING}) + commit(web_repo, "feat: implements it", {"web/src/widget.ts": "export const x = 1\n"}) + profile = Recording(NODE) + + with profiles.registered(profile.toolchain): + problems = check_branch(web_repo, "main", profile=PYTHON, projects=WEB) + + assert problems == [] + assert profile.files == [["src/new.test.ts"]] + + def test_a_renamed_test_file_runs_under_its_new_path_only(repo: Path) -> None: git(repo, "mv", "tests/test_old.py", "tests/test_renamed.py") commit(repo, "test: renames one", {"tests/test_renamed.py": FAILING})