From 93da70279e796e5ad8696604c7775a72b52ee05f Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Tue, 17 Nov 2026 06:48:37 -0700 Subject: [PATCH 1/6] test: Regenerate and commit after a restack The tests come first and fail until the implementation. A unit that edits `agent-assessment/web` also commits the build output (`src/server/web/assets/*-.js`, `src/server/web/index.html`, `src/report/template.html`). Unit: restack-regenerates-generated-files/3 --- src/agent_build_kit/pipeline/generated.py | 18 ++ src/agent_build_kit/pipeline/unit_store.py | 1 + tests/pipeline/test_generated_regenerate.py | 188 ++++++++++++++++++++ tests/standing_causes.py | 1 + 4 files changed, 208 insertions(+) create mode 100644 src/agent_build_kit/pipeline/generated.py create mode 100644 tests/pipeline/test_generated_regenerate.py diff --git a/src/agent_build_kit/pipeline/generated.py b/src/agent_build_kit/pipeline/generated.py new file mode 100644 index 00000000..a2077593 --- /dev/null +++ b/src/agent_build_kit/pipeline/generated.py @@ -0,0 +1,18 @@ +"""Rebuild a repository's generated files after a restack and commit the result.""" + +from collections.abc import Sequence +from pathlib import Path + +from agent_build_kit.config import GeneratedConfig + + +class GeneratedFilesFailed(RuntimeError): + """A regenerate command exited non-zero; `output` is what it printed.""" + + output: str + + +def regenerate(repo: Path, entries: Sequence[GeneratedConfig], *, onto: str) -> None: + """Run each entry whose paths the unit's commits (`onto...HEAD`) touched, in its `cwd`, + and commit what changed as one `chore:` commit. Raises `GeneratedFilesFailed`.""" + raise NotImplementedError diff --git a/src/agent_build_kit/pipeline/unit_store.py b/src/agent_build_kit/pipeline/unit_store.py index cb66bad9..2c2cbc18 100644 --- a/src/agent_build_kit/pipeline/unit_store.py +++ b/src/agent_build_kit/pipeline/unit_store.py @@ -141,6 +141,7 @@ class Cause(StrEnum): TOO_BIG_FOR_TIER = "too_big_for_tier" SUPERSEDED = "superseded" LANDED_OFF_DEFAULT = "landed_off_default" + GENERATED_FILES = "generated_files" BASE_NOT_MOVED = "base_not_moved" diff --git a/tests/pipeline/test_generated_regenerate.py b/tests/pipeline/test_generated_regenerate.py new file mode 100644 index 00000000..0dba89b3 --- /dev/null +++ b/tests/pipeline/test_generated_regenerate.py @@ -0,0 +1,188 @@ +"""After a restack the generated files are rebuilt from the moved sources and committed. + +Real git in temporary repositories; the regenerate command is a small shell script that +appends a line to a log each time it runs, so how often and where it ran is read from the log. +""" + +from __future__ import annotations + +import contextlib +from pathlib import Path + +import pytest + +from agent_build_kit.config import GeneratedConfig, RepoConfig +from agent_build_kit.pipeline.generated import GeneratedFilesFailed, regenerate +from agent_build_kit.pipeline.restack import move_branch_onto, resolved_move +from agent_build_kit.pipeline.unit_store import Cause +from agent_build_kit.pipeline.units import HELD +from agent_build_kit.pipeline.wiring import build_restack_onto +from tests.factories import commit_files, git, init_repo, unit +from tests.pipeline.test_restack_onto_before_push import BASE, BRANCH, Repos, Resolver, stack + +SUBJECT = "chore: Rebuild the generated files after restacking onto main" + + +def rebuilding( + log: Path, *, paths: tuple[str, ...], writes: str, cwd: str = "." +) -> GeneratedConfig: + """An entry whose command logs where it ran, then writes `writes` (relative to its cwd).""" + script = f'echo "ran in $(pwd)" >> "{log}"; printf "rebuilt\\n" > {writes}' + return GeneratedConfig(paths=paths, command=("sh", "-c", script), cwd=cwd) + + +def failing(log: Path, *, paths: tuple[str, ...]) -> GeneratedConfig: + script = f'echo "ran" >> "{log}"; echo "bundler exploded" >&2; exit 1' + return GeneratedConfig(paths=paths, command=("sh", "-c", script)) + + +def runs(log: Path) -> list[str]: + return log.read_text().splitlines() if log.exists() else [] + + +def restacked(tmp_path: Path, *, unit_files: dict[str, str]) -> Path: + """A repository whose `spec/c/2` sits on `main`'s first commit, one commit of the + unit's `unit_files` ahead, and whose `main` has since moved on.""" + repo = init_repo(tmp_path / "repo") + commit_files(repo, "start", {"src/app.py": "v = 0\n", "web/out/bundle.js": "start\n"}) + git(repo, "checkout", "-q", "-b", "spec/c/2") + commit_files(repo, "the unit's work", unit_files) + git(repo, "checkout", "-q", "main") + commit_files(repo, "the base moves", {"base.txt": "base\n"}) + git(repo, "checkout", "-q", "spec/c/2") + git(repo, "rebase", "-q", "main") + return repo + + +def subjects(repo: Path, onto: str = "main") -> list[str]: + return git(repo, "log", "--format=%s", f"{onto}..HEAD").splitlines() + + +def test_a_unit_touching_a_generated_path_rebuilds_once_in_the_entrys_cwd(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "edited\n"}) + log = tmp_path / "log" + entry = rebuilding(log, paths=("web/out/**",), writes="out/bundle.js", cwd="web") + + regenerate(repo, [entry], onto="main") + + assert runs(log) == [f"ran in {(repo / 'web').resolve()}"] + assert (repo / "web/out/bundle.js").read_text() == "rebuilt\n" + + +def test_the_rebuilt_files_are_committed_as_one_chore_commit(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "edited\n"}) + entry = rebuilding(tmp_path / "log", paths=("web/out/**",), writes="web/out/bundle.js") + + regenerate(repo, [entry], onto="main") + + assert subjects(repo) == [SUBJECT, "the unit's work"] + assert git(repo, "status", "--porcelain") == "" + assert git(repo, "show", "HEAD:web/out/bundle.js") == "rebuilt" + + +def test_no_commit_is_added_when_the_rebuilt_output_is_unchanged(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "rebuilt\n"}) + log = tmp_path / "log" + entry = rebuilding(log, paths=("web/out/**",), writes="web/out/bundle.js") + + regenerate(repo, [entry], onto="main") + + assert len(runs(log)) == 1, "the command ran, and found nothing to change" + assert subjects(repo) == ["the unit's work"] + + +def test_a_unit_touching_no_generated_path_does_not_run_the_command(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"src/app.py": "v = 1\n"}) + log = tmp_path / "log" + entry = rebuilding(log, paths=("web/out/**",), writes="web/out/bundle.js") + + regenerate(repo, [entry], onto="main") + + assert runs(log) == [] + assert subjects(repo) == ["the unit's work"] + + +def test_a_failing_command_raises_with_its_output(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "edited\n"}) + + with pytest.raises(GeneratedFilesFailed) as raised: + regenerate(repo, [failing(tmp_path / "log", paths=("web/out/**",))], onto="main") + + assert "bundler exploded" in raised.value.output + assert subjects(repo) == ["the unit's work"] + + +def test_a_failed_regeneration_is_held_under_its_own_cause(tmp_path: Path) -> None: + repos = Repos(tmp_path, edits_shared=False) + store = stack(tmp_path) + repos.advance_trunk(conflicting=False) + entry = failing(tmp_path / "log", paths=("marker.py",)) + config = RepoConfig(path=repos.tree, generated=[entry]) + + with contextlib.suppress(GeneratedFilesFailed): + build_restack_onto(store, repo=config)( + tree=repos.tree, branch=BRANCH, base=BASE, unit=unit(), resolve=False + ) + + held = store.get(unit().id) + assert (held.state, held.cause) == (HELD, Cause.GENERATED_FILES) + assert "bundler exploded" in held.note + + +# The three restack paths each rebuild the generated files exactly once. A unit that adds +# `marker.py` touches it; the entry declares it generated. +def marker_entry(log: Path) -> GeneratedConfig: + return rebuilding(log, paths=("marker.py",), writes="marker.py") + + +def test_a_plain_restack_rebuilds_once(tmp_path: Path) -> None: + repos = Repos(tmp_path, edits_shared=False) + old_base = git(repos.tree, "rev-parse", BASE) + repos.advance_trunk(conflicting=False) + log = tmp_path / "log" + config = RepoConfig(path=repos.tree, generated=[marker_entry(log)]) + + move_branch_onto(repos.tree, BRANCH, new_base=BASE, old_base=old_base, repo_config=config) + + assert len(runs(log)) == 1 + assert git(repos.tree, "log", "-1", "--format=%s").startswith("chore: Rebuild the generated") + + +def test_the_build_workers_moved_base_restack_rebuilds_once(tmp_path: Path) -> None: + repos = Repos(tmp_path, edits_shared=False) + store = stack(tmp_path) + repos.advance_trunk(conflicting=False) + log = tmp_path / "log" + config = RepoConfig(path=repos.tree, generated=[marker_entry(log)]) + + moved = build_restack_onto(store, repo=config)( + tree=repos.tree, branch=BRANCH, base=BASE, unit=unit(), resolve=False + ) + + assert moved is not None + assert len(runs(log)) == 1 + assert git(repos.tree, "log", "-1", "--format=%s").startswith("chore: Rebuild the generated") + + +def test_the_resolution_lane_restack_rebuilds_once(tmp_path: Path) -> None: + repos = Repos(tmp_path, edits_shared=False) + old_base = git(repos.tree, "rev-parse", BASE) + repos.advance_trunk(conflicting=False) + log = tmp_path / "log" + config = RepoConfig(path=repos.tree, generated=[marker_entry(log)]) + + resolved_move( + repos.tree, + BRANCH, + new_base=BASE, + old_base=old_base, + moving_unit="feature/2", + moving_intent="add the marker", + onto_unit="feature/1", + onto_intent="move the trunk", + resolve=Resolver(), + repo_config=config, + ) + + assert len(runs(log)) == 1 + assert git(repos.tree, "log", "-1", "--format=%s").startswith("chore: Rebuild the generated") diff --git a/tests/standing_causes.py b/tests/standing_causes.py index 7411f9b4..84864193 100644 --- a/tests/standing_causes.py +++ b/tests/standing_causes.py @@ -18,6 +18,7 @@ Cause.MANUAL_STEP, Cause.TOO_BIG_FOR_TIER, Cause.LANDED_OFF_DEFAULT, + Cause.GENERATED_FILES, Cause.BASE_NOT_MOVED, Cause.GATED, Cause.UPSTREAM_WENT_BACK, From c653a9f1c4b7421f7ec1f8e2a9baefb06d10df28 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Tue, 17 Nov 2026 06:51:08 -0700 Subject: [PATCH 2/6] feat: Regenerate and commit after a restack With this in, the tests of the previous commit pass. A unit that edits `agent-assessment/web` also commits the build output (`src/server/web/assets/*-.js`, `src/server/web/index.html`, `src/report/template.html`). Unit: restack-regenerates-generated-files/3 --- CHANGELOG.md | 5 ++- src/agent_build_kit/pipeline/generated.py | 40 ++++++++++++++++++++++- src/agent_build_kit/pipeline/restack.py | 10 ++++++ src/agent_build_kit/pipeline/wiring.py | 11 +++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d5782d0..95b26af4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,10 @@ - A restack now settles conflicts in files a repository's `generated` setting covers by taking the base's version (or removing the file where the base has none), without asking the resolver; only source conflicts reach it, and its prompt names the generated paths as - handled. Reading a conflict no longer fails on a path missing from the working tree. + handled. Reading a conflict no longer fails on a path missing from the working tree. After + the restack, each `generated` entry whose paths the unit touched is run in its `cwd` and the + result committed as one `chore:` commit; a failing command holds the unit under the + `generated_files` cause with its output in the note. - The red check and the push gate no longer give the test runner a test file the tests commit deletes or renames away. Before, the runner was named a path that no longer existed and diff --git a/src/agent_build_kit/pipeline/generated.py b/src/agent_build_kit/pipeline/generated.py index a2077593..75ac42e2 100644 --- a/src/agent_build_kit/pipeline/generated.py +++ b/src/agent_build_kit/pipeline/generated.py @@ -1,9 +1,12 @@ """Rebuild a repository's generated files after a restack and commit the result.""" +import subprocess from collections.abc import Sequence +from fnmatch import fnmatch from pathlib import Path from agent_build_kit.config import GeneratedConfig +from agent_build_kit.pipeline.shell import git class GeneratedFilesFailed(RuntimeError): @@ -11,8 +14,43 @@ class GeneratedFilesFailed(RuntimeError): output: str + def __init__(self, output: str) -> None: + super().__init__(f"regenerating the generated files failed:\n{output}") + self.output = output + + +def _matches(name: str, patterns: Sequence[str]) -> bool: + return any(fnmatch(name, pattern) for pattern in patterns) + def regenerate(repo: Path, entries: Sequence[GeneratedConfig], *, onto: str) -> None: """Run each entry whose paths the unit's commits (`onto...HEAD`) touched, in its `cwd`, and commit what changed as one `chore:` commit. Raises `GeneratedFilesFailed`.""" - raise NotImplementedError + touched = git(repo, "diff", "--name-only", f"{onto}...HEAD").stdout.split() + ran = False + for entry in entries: + if not any(_matches(name, entry.paths) for name in touched): + continue + done = subprocess.run( + list(entry.command), + cwd=repo / entry.cwd, + capture_output=True, + text=True, + check=False, + ) + if done.returncode != 0: + raise GeneratedFilesFailed((done.stdout + done.stderr).strip()) + ran = True + if not ran: + return + git(repo, "add", "-A") + if not git(repo, "diff", "--cached", "--name-only").stdout.strip(): + return + git( + repo, + "commit", + "--no-verify", + "-q", + "-m", + f"chore: Rebuild the generated files after restacking onto {onto}", + ) diff --git a/src/agent_build_kit/pipeline/restack.py b/src/agent_build_kit/pipeline/restack.py index 0cbd1139..1abea15a 100644 --- a/src/agent_build_kit/pipeline/restack.py +++ b/src/agent_build_kit/pipeline/restack.py @@ -46,6 +46,7 @@ changelog_path, resolver_changelog_rule, ) +from agent_build_kit.pipeline.generated import regenerate from agent_build_kit.pipeline.git_output import ( PUSH_OPERATION, GitLockFailed, @@ -527,6 +528,12 @@ def _land_detached(repo: Path, branch: str, *, started: str) -> Moved: return Moved(sha=head) +def _regenerate(repo: Path, config: RepoConfig | None, *, onto: str) -> None: + """Rebuild the repo's generated files once the rebase has finished. Inside the turn.""" + if config is not None and config.generated: + regenerate(repo, config.generated, onto=onto) + + def _separate_changelog(repo: Path, config: RepoConfig | None) -> None: """Put back the blank line a union merge took from between two changelog bullets, in a commit of its own on the replayed branch. Inside the turn, with the rebase finished.""" @@ -685,6 +692,7 @@ def move_branch_onto( ) if result.returncode == 0: _separate_changelog(repo, repo_config) + _regenerate(repo, repo_config, onto=new_base) if detached: return _land_detached(repo, branch, started=started) return Moved(sha=git(repo, "rev-parse", branch).stdout.strip()) @@ -695,6 +703,7 @@ def move_branch_onto( done, result = _settle_generated(repo, result, patterns, settled) if done: _separate_changelog(repo, repo_config) + _regenerate(repo, repo_config, onto=new_base) if detached: return _land_detached(repo, branch, started=started) return Moved(sha=git(repo, "rev-parse", branch).stdout.strip()) @@ -828,6 +837,7 @@ def move_branch_onto( tried="\n".join(failures), ) _separate_changelog(repo, repo_config) + _regenerate(repo, repo_config, onto=new_base) if detached: landed = _land_detached(repo, branch, started=started) return Moved(sha=landed.sha, resolved=resolved) diff --git a/src/agent_build_kit/pipeline/wiring.py b/src/agent_build_kit/pipeline/wiring.py index 6d772ce7..18ac7133 100644 --- a/src/agent_build_kit/pipeline/wiring.py +++ b/src/agent_build_kit/pipeline/wiring.py @@ -129,6 +129,7 @@ record_storm, ) from agent_build_kit.pipeline.gateway_usage import Spend, attribution, configured_source +from agent_build_kit.pipeline.generated import GeneratedFilesFailed from agent_build_kit.pipeline.git_output import ( FetchFailed, TransportFailure, @@ -226,6 +227,7 @@ ) from agent_build_kit.pipeline.units import ( BUILDS_NOTHING, + HELD, IN_REVIEW, MERGED, PLANNED, @@ -4843,6 +4845,15 @@ def restack_onto( **asking, ) ) + except GeneratedFilesFailed as error: + store.set_state( + unit.id, + HELD, + note=f"rebuilding the generated files after restacking onto {base} " + f"failed:\n{cut_head(error.output, 1000, 'char')}", + cause=Cause.GENERATED_FILES, + ) + raise except RestackConflict as error: # Not a failure: what the resolver's steps could not settle goes to the adapt # step, which ports the work by hand. It needs From 71c50bb764d56453bc34752ab3e28765ed298ad6 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Tue, 17 Nov 2026 08:56:30 -0700 Subject: [PATCH 3/6] test: Documents and skills The tests come first and fail until the implementation. Unit: restack-regenerates-generated-files/3 --- tests/docs/test_generated_docs.py | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/docs/test_generated_docs.py diff --git a/tests/docs/test_generated_docs.py b/tests/docs/test_generated_docs.py new file mode 100644 index 00000000..1d5586b1 --- /dev/null +++ b/tests/docs/test_generated_docs.py @@ -0,0 +1,53 @@ +"""The documents and the `abk-config` skill describe `repos..generated`: its keys, what a +restack does with the paths, the hold cause, and the changelog entry.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agent_build_kit import skills +from tests.prompt_text import flat + +ROOT = Path(__file__).resolve().parents[2] + + +def doc(*parts: str) -> str: + return flat(ROOT.joinpath(*parts).read_text(encoding="utf-8")).lower() + + +def unreleased_section() -> str: + text = ROOT.joinpath("CHANGELOG.md").read_text(encoding="utf-8") + after = text.split("## Unreleased", 1)[1] + return flat(after.split("\n## ", 1)[0]).lower() + + +def test_configuration_documents_the_generated_setting_keys() -> None: + text = doc("docs", "configuration.md") + for term in ("generated", "paths", "command", "cwd"): + assert f"`{term}`" in text + + +def test_configuration_says_what_a_restack_does_with_generated_files() -> None: + text = doc("docs", "configuration.md") + assert "restack" in text + assert "base" in text + assert "`generated_files`" in text + + +def test_configuration_says_a_slash_free_pattern_matches_by_file_name_anywhere() -> None: + text = doc("docs", "configuration.md") + assert "slash-free pattern" in text + assert "file name" in text + + +def test_the_abk_config_skill_mentions_the_generated_setting() -> None: + text = flat(skills.source("abk-config")).lower() + assert "generated:" in text + assert "cwd" in text + + +@pytest.mark.parametrize("term", ["generated", "restack", "`generated_files`"]) +def test_the_changelog_entry_under_unreleased_covers_the_setting(term: str) -> None: + assert term in unreleased_section() From ad406c1aa8321458b3e56719d15005cda17876a7 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Tue, 17 Nov 2026 10:17:41 -0700 Subject: [PATCH 4/6] feat: Documents and skills With this in, the tests of the previous commit pass. A unit that edits `agent-assessment/web` also commits the build output (`src/server/web/assets/*-.js`, `src/server/web/index.html`, `src/report/template.html`). Unit: restack-regenerates-generated-files/3 --- docs/configuration.md | 18 ++++++++++++++++++ src/agent_build_kit/skills/abk-config/SKILL.md | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 516c5c10..71a71c77 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -676,6 +676,24 @@ repos: # ordered: deploy order is derived from # one here to give agents one. # Each must match the build allow-list (the tool # list the build starts with) or the load fails + generated: # `generated`: files built from others, regenerated + # after a restack and not merged + - paths: ["web/dist/**"] # `paths`: patterns; a slash-free pattern matches + # by file name anywhere in the repository (`*.js` + # covers every tracked .js). They also count as + # generated for the size rule and the doctor + command: ["npm", "run", "build"] + # `command`: argv run in `cwd` (default `.`, a + # path inside the repository) after a restack + cwd: web + # A restack settles a conflict in these paths + # by taking the base's version (removing the file + # where the base has none), never asking the + # resolver. Then, if the unit touched a path, + # the command runs once and its changes are + # committed. A failing command holds the unit + # with cause `generated_files`, its output in + # the note changelog: CHANGELOG.md # where the repo keeps its changelog. Set, the # build, rework, resolver and review prompts # carry its convention and unit-test runs diff --git a/src/agent_build_kit/skills/abk-config/SKILL.md b/src/agent_build_kit/skills/abk-config/SKILL.md index 163373ef..647983f1 100644 --- a/src/agent_build_kit/skills/abk-config/SKILL.md +++ b/src/agent_build_kit/skills/abk-config/SKILL.md @@ -365,6 +365,11 @@ repos: # ordered; a task group's [repo] tag is a key here # scope: whole|file|node-id|affected}; `{path}` in the command exactly # when the scope is file or node-id. null follows the profiles, a list # replaces them. Each must match the build allow-list: refused at load + generated: [] # [{paths: [...], command: [argv], cwd: "."}]: files built from others. + # A restack takes the base's version of a conflicted path (no + # resolver), then runs `command` in `cwd` once if the unit touched a + # path and commits the result; a failure holds the unit, cause + # `generated_files`. A slash-free pattern matches by file name anywhere changelog: CHANGELOG.md # the repo's changelog; null = no convention, no check. # Convention: AGENTS.md `## Changelog`, else CLAUDE.md, # else the packaged text. Doctor warns if the file is absent From 6f2e5bf509b2f9a4479d96a875e9542be8cb1bc1 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Tue, 17 Nov 2026 11:12:23 -0700 Subject: [PATCH 5/6] fix: Regenerate and commit generated files after a restack, the generated_files hold cause, and docs (checks, round 1) Unit: restack-regenerates-generated-files/3 --- docs/architecture.md | 2 +- docs/designs/unit-graph.md | 2 +- tests/pipeline/test_unit_causes.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index aa8494b2..2551bbbc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -947,7 +947,7 @@ A standing cause names a situation that lasts until a state change passes anothe `review_escalated_disagreement`, `late_failure`, `needs_human`, `reviewer_hold`, `gated`, `restack_conflict`, `restack_deferred`, `restack_handed_back`, `dirty_worktree`, `attached`, `host_unavailable`, `network_unavailable`, `service_unavailable`, `environment`, `split_held`, `manual_step`, -`too_big_for_tier`, `landed_off_default` (a pull request merged into a branch that is not the default branch) and `base_not_moved` (a pull request whose retarget failed three rounds in a row). +`too_big_for_tier`, `landed_off_default` (a pull request merged into a branch that is not the default branch) `base_not_moved` (a pull request whose retarget failed three rounds in a row) and `generated_files` (a repository's `generated` command failed after a restack). An event cause names a transition that has happened and is over: `requeued`, `released`, `adopted`, `merged`, `auto_landed`, `closed`, `retired`, `superseded` and `failed`. Recording an event cause ends a standing one explicitly. A state change the store makes on its own account (a replan's demotion and diff --git a/docs/designs/unit-graph.md b/docs/designs/unit-graph.md index 29fda969..e634eaa4 100644 --- a/docs/designs/unit-graph.md +++ b/docs/designs/unit-graph.md @@ -101,7 +101,7 @@ until a state change passes another cause: `rework`, `base_changed`, `upstream_w `depth`, `toolchain`, `review_escalated_class`, `review_escalated_disagreement`, `late_failure`, `needs_human`, `reviewer_hold`, `gated`, `restack_conflict`, `restack_deferred`, `restack_handed_back`, `dirty_worktree`, `attached`, `host_unavailable`, `network_unavailable`, `service_unavailable`, -`environment`, `split_held`, `manual_step`, `too_big_for_tier`, `landed_off_default` and `base_not_moved`. The event causes (`EVENT_CAUSES`) name a transition that is over: +`environment`, `split_held`, `manual_step`, `too_big_for_tier`, `landed_off_default`, `generated_files` and `base_not_moved`. The event causes (`EVENT_CAUSES`) name a transition that is over: `requeued`, `released`, `adopted`, `merged`, `auto_landed`, `closed`, `retired`, `superseded` and `failed`. An event cause ends a standing one explicitly. A replan carries a standing cause through the demotion and the promotion (and a vanished change and a join do the same), marking the entry `carried`, and a caller's state diff --git a/tests/pipeline/test_unit_causes.py b/tests/pipeline/test_unit_causes.py index dadb3e65..5a1d5a49 100644 --- a/tests/pipeline/test_unit_causes.py +++ b/tests/pipeline/test_unit_causes.py @@ -70,6 +70,7 @@ def test_the_causes_are_the_fixed_set_the_design_names() -> None: "too_big_for_tier", "superseded", "landed_off_default", + "generated_files", "base_not_moved", } From 141349c3f84709a3c96ea362e1d27207181f2039 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Tue, 17 Nov 2026 12:00:51 -0700 Subject: [PATCH 6/6] fix: Regenerate and commit generated files after a restack, the generated_files hold cause, and docs (review round 1) Unit: restack-regenerates-generated-files/3 --- docs/architecture.md | 2 +- src/agent_build_kit/graph/nodes.py | 11 ++ src/agent_build_kit/pipeline/events.py | 10 ++ src/agent_build_kit/pipeline/generated.py | 74 +++++++++--- src/agent_build_kit/pipeline/restack.py | 28 +++-- src/agent_build_kit/pipeline/wiring.py | 11 -- tests/pipeline/test_events_restack_scope.py | 17 ++- tests/pipeline/test_generated_regenerate.py | 126 +++++++++++++++++++- 8 files changed, 233 insertions(+), 46 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 2551bbbc..dcdbc5f9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -947,7 +947,7 @@ A standing cause names a situation that lasts until a state change passes anothe `review_escalated_disagreement`, `late_failure`, `needs_human`, `reviewer_hold`, `gated`, `restack_conflict`, `restack_deferred`, `restack_handed_back`, `dirty_worktree`, `attached`, `host_unavailable`, `network_unavailable`, `service_unavailable`, `environment`, `split_held`, `manual_step`, -`too_big_for_tier`, `landed_off_default` (a pull request merged into a branch that is not the default branch) `base_not_moved` (a pull request whose retarget failed three rounds in a row) and `generated_files` (a repository's `generated` command failed after a restack). +`too_big_for_tier`, `landed_off_default` (a pull request merged into a branch that is not the default branch), `base_not_moved` (a pull request whose retarget failed three rounds in a row) and `generated_files` (a repository's `generated` command failed after a restack). An event cause names a transition that has happened and is over: `requeued`, `released`, `adopted`, `merged`, `auto_landed`, `closed`, `retired`, `superseded` and `failed`. Recording an event cause ends a standing one explicitly. A state change the store makes on its own account (a replan's demotion and diff --git a/src/agent_build_kit/graph/nodes.py b/src/agent_build_kit/graph/nodes.py index 042b014d..fc4f0502 100644 --- a/src/agent_build_kit/graph/nodes.py +++ b/src/agent_build_kit/graph/nodes.py @@ -92,6 +92,7 @@ waiting_note, ) from agent_build_kit.pipeline.gateway_usage import attribution +from agent_build_kit.pipeline.generated import GeneratedFilesFailed, hold_note from agent_build_kit.pipeline.group_mode import GroupMode from agent_build_kit.pipeline.manual_steps import unmet_steps from agent_build_kit.pipeline.metric_records import emit_metric @@ -1301,6 +1302,13 @@ def prepare(self, state: UnitRun) -> Update: **remembered, **self.hold(PLANNED, note, note, cause=Cause.RESTACK_DEFERRED), } + except GeneratedFilesFailed as error: + note = hold_note(base, error) + return { + **FRESH, + **remembered, + **self.hold(HELD, note, note, cause=Cause.GENERATED_FILES), + } except Exception as error: # noqa: BLE001 why = f"restack onto {base} conflicted: {error}" self.append_feedback(why) @@ -3139,6 +3147,9 @@ def restack_now(self, state: UnitRun, point: str, *, find_base: bool = True) -> ) except (AgentRateLimited, AgentInterrupted, NetworkUnavailable, ServiceUnavailable): raise + except GeneratedFilesFailed as error: + note = hold_note(base, error) + return self.hold(HELD, note, note, cause=Cause.GENERATED_FILES) except Exception as error: # noqa: BLE001 return self.rebase( state, diff --git a/src/agent_build_kit/pipeline/events.py b/src/agent_build_kit/pipeline/events.py index e5cc3c78..34b9cd35 100644 --- a/src/agent_build_kit/pipeline/events.py +++ b/src/agent_build_kit/pipeline/events.py @@ -66,6 +66,7 @@ is_fix_change, waiting_note, ) +from agent_build_kit.pipeline.generated import GeneratedFilesFailed, hold_note from agent_build_kit.pipeline.landing_check import Landing, merged_on_branch, merged_on_default from agent_build_kit.pipeline.landing_intent import clear_intent, has_intent from agent_build_kit.pipeline.pr_poller import ( # noqa: F401 — FAILING_CHECKS_REASON re-exported @@ -2749,6 +2750,15 @@ def restack( cause=Cause.RESTACK_CONFLICT, ) return RestackOutcome.CONFLICT + except GeneratedFilesFailed as error: + # The branch is back where it was; a person fixes the command and requeues. + store.set_state( + child.id, + HELD, + note=hold_note(new_base, error), + cause=Cause.GENERATED_FILES, + ) + return RestackOutcome.DEFERRED except AGENT_UNAVAILABLE as error: # The resolver could not run, which says nothing about the branch. # Not raised on: the merge is handled once, and the parent's branch diff --git a/src/agent_build_kit/pipeline/generated.py b/src/agent_build_kit/pipeline/generated.py index 75ac42e2..08fe4c37 100644 --- a/src/agent_build_kit/pipeline/generated.py +++ b/src/agent_build_kit/pipeline/generated.py @@ -2,15 +2,17 @@ import subprocess from collections.abc import Sequence -from fnmatch import fnmatch from pathlib import Path +from agent_build_kit.budget import cut_head from agent_build_kit.config import GeneratedConfig +from agent_build_kit.pipeline.check_runner import COMMAND_TIMEOUT_SECONDS from agent_build_kit.pipeline.shell import git +from agent_build_kit.pipeline.unit_size import is_generated class GeneratedFilesFailed(RuntimeError): - """A regenerate command exited non-zero; `output` is what it printed.""" + """A regenerate command could not run, timed out or exited non-zero; `output` says why.""" output: str @@ -19,31 +21,69 @@ def __init__(self, output: str) -> None: self.output = output -def _matches(name: str, patterns: Sequence[str]) -> bool: - return any(fnmatch(name, pattern) for pattern in patterns) +def hold_note(onto: str, error: GeneratedFilesFailed) -> str: + """The note a unit is held with when its generated files could not be rebuilt: the one + text every restack path records under the `generated_files` cause.""" + return ( + f"rebuilding the generated files after restacking onto {onto} failed:\n" + f"{cut_head(error.output, 1000, 'char')}" + ) + + +def _changed_under(repo: Path, patterns: Sequence[str]) -> list[str]: + """Paths the working tree modifies, deletes or adds that `patterns` cover.""" + listed = git( + repo, "ls-files", "-z", "--modified", "--deleted", "--others", "--exclude-standard" + ).stdout + return sorted({name for name in listed.split("\0") if name and is_generated(name, patterns)}) -def regenerate(repo: Path, entries: Sequence[GeneratedConfig], *, onto: str) -> None: +def discard_output(repo: Path, entries: Sequence[GeneratedConfig]) -> None: + """Remove what a failed command left under its entries' paths: edits and new files.""" + left = _changed_under(repo, [pattern for entry in entries for pattern in entry.paths]) + if left: + git(repo, "checkout", "-q", "HEAD", "--", *left, check=False) + git(repo, "clean", "-fdq", "--", *left, check=False) + + +def regenerate( + repo: Path, + entries: Sequence[GeneratedConfig], + *, + onto: str, + timeout: float = COMMAND_TIMEOUT_SECONDS, +) -> None: """Run each entry whose paths the unit's commits (`onto...HEAD`) touched, in its `cwd`, - and commit what changed as one `chore:` commit. Raises `GeneratedFilesFailed`.""" + and commit what changed under those entries' paths as one `chore:` commit. A command that + cannot start, runs past `timeout` seconds or exits non-zero raises `GeneratedFilesFailed`.""" touched = git(repo, "diff", "--name-only", f"{onto}...HEAD").stdout.split() - ran = False + ran: list[GeneratedConfig] = [] for entry in entries: - if not any(_matches(name, entry.paths) for name in touched): + if not any(is_generated(name, entry.paths) for name in touched): continue - done = subprocess.run( - list(entry.command), - cwd=repo / entry.cwd, - capture_output=True, - text=True, - check=False, - ) + shown = " ".join(entry.command) + try: + done = subprocess.run( + list(entry.command), + cwd=repo / entry.cwd, + capture_output=True, + text=True, + check=False, + timeout=timeout, + ) + except OSError as error: + raise GeneratedFilesFailed(f"`{shown}` could not be run: {error}") from error + except subprocess.TimeoutExpired as error: + raise GeneratedFilesFailed(f"`{shown}` timed out after {timeout:g}s") from error if done.returncode != 0: raise GeneratedFilesFailed((done.stdout + done.stderr).strip()) - ran = True + ran.append(entry) if not ran: return - git(repo, "add", "-A") + changed = _changed_under(repo, [pattern for entry in ran for pattern in entry.paths]) + if not changed: + return + git(repo, "add", "-A", "--", *changed) if not git(repo, "diff", "--cached", "--name-only").stdout.strip(): return git( diff --git a/src/agent_build_kit/pipeline/restack.py b/src/agent_build_kit/pipeline/restack.py index 1abea15a..66dd0b7b 100644 --- a/src/agent_build_kit/pipeline/restack.py +++ b/src/agent_build_kit/pipeline/restack.py @@ -46,7 +46,7 @@ changelog_path, resolver_changelog_rule, ) -from agent_build_kit.pipeline.generated import regenerate +from agent_build_kit.pipeline.generated import GeneratedFilesFailed, discard_output, regenerate from agent_build_kit.pipeline.git_output import ( PUSH_OPERATION, GitLockFailed, @@ -528,10 +528,20 @@ def _land_detached(repo: Path, branch: str, *, started: str) -> Moved: return Moved(sha=head) -def _regenerate(repo: Path, config: RepoConfig | None, *, onto: str) -> None: - """Rebuild the repo's generated files once the rebase has finished. Inside the turn.""" - if config is not None and config.generated: +def _regenerate(repo: Path, config: RepoConfig | None, *, onto: str, started: str) -> None: + """Rebuild the repo's generated files once the rebase has finished. Inside the turn. + + A failed rebuild puts the head back at `started` (the branch's, or a detached tree's, head + before the move) and removes what the command wrote under its entries' paths, so the move + is replayed, and the rebuild tried again, the next time it is asked for.""" + if config is None or not config.generated: + return + try: regenerate(repo, config.generated, onto=onto) + except GeneratedFilesFailed: + git(repo, "reset", "-q", "--hard", started) + discard_output(repo, config.generated) + raise def _separate_changelog(repo: Path, config: RepoConfig | None) -> None: @@ -665,7 +675,9 @@ def move_branch_onto( with the rebase aborted. With `detached`, `repo` is a worktree on no branch: the rebase runs on its detached head and the branch is moved at the end by compare-and-swap, unless another worktree has it checked out - (`BranchCheckedOut`). + (`BranchCheckedOut`). The regenerate commands of the repository's `generated` entries run + inside the turn, once the rebase has finished, so a long build holds the turn; one that + fails puts the branch back where the move started and raises `GeneratedFilesFailed`. `repo_config` (else the `context`'s) names the repo whose `changelog` setting decides which file is separated once the branch is replayed; with neither, the default name. @@ -692,7 +704,7 @@ def move_branch_onto( ) if result.returncode == 0: _separate_changelog(repo, repo_config) - _regenerate(repo, repo_config, onto=new_base) + _regenerate(repo, repo_config, onto=new_base, started=started) if detached: return _land_detached(repo, branch, started=started) return Moved(sha=git(repo, "rev-parse", branch).stdout.strip()) @@ -703,7 +715,7 @@ def move_branch_onto( done, result = _settle_generated(repo, result, patterns, settled) if done: _separate_changelog(repo, repo_config) - _regenerate(repo, repo_config, onto=new_base) + _regenerate(repo, repo_config, onto=new_base, started=started) if detached: return _land_detached(repo, branch, started=started) return Moved(sha=git(repo, "rev-parse", branch).stdout.strip()) @@ -837,7 +849,7 @@ def move_branch_onto( tried="\n".join(failures), ) _separate_changelog(repo, repo_config) - _regenerate(repo, repo_config, onto=new_base) + _regenerate(repo, repo_config, onto=new_base, started=started) if detached: landed = _land_detached(repo, branch, started=started) return Moved(sha=landed.sha, resolved=resolved) diff --git a/src/agent_build_kit/pipeline/wiring.py b/src/agent_build_kit/pipeline/wiring.py index 18ac7133..6d772ce7 100644 --- a/src/agent_build_kit/pipeline/wiring.py +++ b/src/agent_build_kit/pipeline/wiring.py @@ -129,7 +129,6 @@ record_storm, ) from agent_build_kit.pipeline.gateway_usage import Spend, attribution, configured_source -from agent_build_kit.pipeline.generated import GeneratedFilesFailed from agent_build_kit.pipeline.git_output import ( FetchFailed, TransportFailure, @@ -227,7 +226,6 @@ ) from agent_build_kit.pipeline.units import ( BUILDS_NOTHING, - HELD, IN_REVIEW, MERGED, PLANNED, @@ -4845,15 +4843,6 @@ def restack_onto( **asking, ) ) - except GeneratedFilesFailed as error: - store.set_state( - unit.id, - HELD, - note=f"rebuilding the generated files after restacking onto {base} " - f"failed:\n{cut_head(error.output, 1000, 'char')}", - cause=Cause.GENERATED_FILES, - ) - raise except RestackConflict as error: # Not a failure: what the resolver's steps could not settle goes to the adapt # step, which ports the work by hand. It needs diff --git a/tests/pipeline/test_events_restack_scope.py b/tests/pipeline/test_events_restack_scope.py index dee8f763..6834f736 100644 --- a/tests/pipeline/test_events_restack_scope.py +++ b/tests/pipeline/test_events_restack_scope.py @@ -32,10 +32,21 @@ def __call__(self, **kwargs: Any) -> tuple[bool, str]: def restacking( - installation: Installation, moved: Moved, *, name: str = "feature", diff_changes: bool = False + installation: Installation, + moved: Moved | Exception, + *, + name: str = "feature", + diff_changes: bool = False, ): """The handler over unit `name`, approved at `approved`, and a way to run it; returns the - store, the unit-test it was given, the pushes and the run.""" + store, the unit-test it was given, the pushes and the run. A `moved` that is an exception + is raised by the move.""" + + def move(*args: Any, **kwargs: Any) -> Moved: + if isinstance(moved, Exception): + raise moved + return moved + child = change_with_unit(installation, name, f"{name}/1").model_copy( update={"pr": 2, "branch": f"spec/{name}/1", "approved": "approved"} ) @@ -47,7 +58,7 @@ def restacking( repos={"app": installation.checkouts["app"]}, store=store, root=installation.root, - move=lambda *a, **k: moved, + move=move, unit_test=unit_test, push=lambda *a, **k: pushed.append("pushed") or "newsha", retarget=lambda *a, **k: None, diff --git a/tests/pipeline/test_generated_regenerate.py b/tests/pipeline/test_generated_regenerate.py index 0dba89b3..d06740e1 100644 --- a/tests/pipeline/test_generated_regenerate.py +++ b/tests/pipeline/test_generated_regenerate.py @@ -6,18 +6,23 @@ from __future__ import annotations -import contextlib from pathlib import Path +from typing import Any import pytest from agent_build_kit.config import GeneratedConfig, RepoConfig +from agent_build_kit.installation import Installation from agent_build_kit.pipeline.generated import GeneratedFilesFailed, regenerate from agent_build_kit.pipeline.restack import move_branch_onto, resolved_move from agent_build_kit.pipeline.unit_store import Cause from agent_build_kit.pipeline.units import HELD from agent_build_kit.pipeline.wiring import build_restack_onto from tests.factories import commit_files, git, init_repo, unit +from tests.graph.test_build_path import build +from tests.graph.test_remaining_paths import fresh +from tests.pipeline.test_events_restack_scope import restacking +from tests.pipeline.test_flake_fix_change import code_repo from tests.pipeline.test_restack_onto_before_push import BASE, BRANCH, Repos, Resolver, stack SUBJECT = "chore: Rebuild the generated files after restacking onto main" @@ -112,21 +117,130 @@ def test_a_failing_command_raises_with_its_output(tmp_path: Path) -> None: assert subjects(repo) == ["the unit's work"] -def test_a_failed_regeneration_is_held_under_its_own_cause(tmp_path: Path) -> None: +def test_the_build_workers_restack_lets_a_failed_rebuild_reach_its_caller(tmp_path: Path) -> None: repos = Repos(tmp_path, edits_shared=False) store = stack(tmp_path) repos.advance_trunk(conflicting=False) - entry = failing(tmp_path / "log", paths=("marker.py",)) - config = RepoConfig(path=repos.tree, generated=[entry]) + config = RepoConfig( + path=repos.tree, generated=[failing(tmp_path / "log", paths=("marker.py",))] + ) - with contextlib.suppress(GeneratedFilesFailed): + with pytest.raises(GeneratedFilesFailed, match="bundler exploded"): build_restack_onto(store, repo=config)( tree=repos.tree, branch=BRANCH, base=BASE, unit=unit(), resolve=False ) - held = store.get(unit().id) + +def test_a_failed_rebuild_in_the_prepare_step_ends_the_unit_held_not_failed( + tmp_path: Path, +) -> None: + recorder = fresh(tmp_path) + + def rebuild_fails(**kw: Any) -> None: + raise GeneratedFilesFailed("bundler exploded") + + outcome = build( + tmp_path, + recorder, + branch_commits=lambda cwd, base: 2 + recorder.made, + restack_onto=rebuild_fails, + ) + + stored = recorder.store.get(unit().id) + assert outcome.status == "held" + assert (stored.state, stored.cause) == (HELD, Cause.GENERATED_FILES) + assert "bundler exploded" in stored.note + assert "push" not in recorder.events and "pr" not in recorder.events + + +def test_a_failed_rebuild_in_the_merge_handlers_restack_holds_the_unit( + installation: Installation, +) -> None: + code_repo(installation) + store, _, pushed, run = restacking(installation, GeneratedFilesFailed("bundler exploded")) + + run() + + held = store.get("feature/1") assert (held.state, held.cause) == (HELD, Cause.GENERATED_FILES) assert "bundler exploded" in held.note + assert pushed == [] + + +def started_at(repos: Repos) -> str: + return git(repos.tree, "rev-parse", BRANCH) + + +def test_a_failed_rebuild_puts_the_branch_back_and_the_next_move_rebuilds( + tmp_path: Path, +) -> None: + repos = Repos(tmp_path, edits_shared=False) + old_base = git(repos.tree, "rev-parse", BASE) + repos.advance_trunk(conflicting=False) + before = started_at(repos) + script = 'echo "half" > marker.py; echo "bundler exploded" >&2; exit 1' + broken = GeneratedConfig(paths=("marker.py",), command=("sh", "-c", script)) + + with pytest.raises(GeneratedFilesFailed, match="bundler exploded"): + move_branch_onto( + repos.tree, + BRANCH, + new_base=BASE, + old_base=old_base, + repo_config=RepoConfig(path=repos.tree, generated=[broken]), + ) + + assert started_at(repos) == before + assert git(repos.tree, "status", "--porcelain") == "" + + log = tmp_path / "log" + move_branch_onto( + repos.tree, + BRANCH, + new_base=BASE, + old_base=old_base, + repo_config=RepoConfig(path=repos.tree, generated=[marker_entry(log)]), + ) + + assert len(runs(log)) == 1 + assert git(repos.tree, "log", "-1", "--format=%s").startswith("chore: Rebuild the generated") + + +def test_a_bare_file_name_pattern_matches_the_file_anywhere(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "edited\n"}) + log = tmp_path / "log" + entry = rebuilding(log, paths=("bundle.js",), writes="web/out/bundle.js") + + regenerate(repo, [entry], onto="main") + + assert len(runs(log)) == 1 + + +def test_only_changes_under_the_entrys_paths_are_committed(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "edited\n"}) + script = 'printf "x\\n" > web/out/bundle.js; printf "cache\\n" > web/cache.txt' + entry = GeneratedConfig(paths=("web/out/**",), command=("sh", "-c", script)) + + regenerate(repo, [entry], onto="main") + + assert git(repo, "show", "--name-only", "--format=", "HEAD") == "web/out/bundle.js" + assert git(repo, "status", "--porcelain") == "?? web/cache.txt" + + +def test_a_command_that_is_not_installed_raises_naming_it(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "edited\n"}) + entry = GeneratedConfig(paths=("web/out/**",), command=("no-such-bundler-here", "build")) + + with pytest.raises(GeneratedFilesFailed, match="no-such-bundler-here build"): + regenerate(repo, [entry], onto="main") + + +def test_a_command_running_past_the_timeout_raises(tmp_path: Path) -> None: + repo = restacked(tmp_path, unit_files={"web/out/bundle.js": "edited\n"}) + entry = GeneratedConfig(paths=("web/out/**",), command=("sleep", "60")) + + with pytest.raises(GeneratedFilesFailed, match="timed out"): + regenerate(repo, [entry], onto="main", timeout=0.05) # The three restack paths each rebuild the generated files exactly once. A unit that adds