From cc85ea104575c4c732301e0ce3732adb8abf19e6 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Sat, 14 Nov 2026 08:34:52 -0700 Subject: [PATCH 1/5] test: A failing test with a plain error is accepted as red The tests come first and fail until the implementation. The Node profile judges a tests commit red only when a failing test's JUnit `type` is `AssertionError`, or its message is exactly `not implemented`. Unit: js-red-check-accepts-a-thrown-error/1 --- tests/fixtures/external/record_vitest.py | 25 +++++++++++ .../external/vitest/error_and_passing.xml | 15 +++++++ .../fixtures/external/vitest/plain_error.xml | 13 ++++++ .../vitest/reference_error_thrown.xml | 13 ++++++ .../external/vitest/syntax_error_thrown.xml | 13 ++++++ .../external/vitest/type_error_thrown.xml | 13 ++++++ tests/profiles/test_node_npm_interpret_red.py | 45 +++++++++++++++++++ 7 files changed, 137 insertions(+) create mode 100644 tests/fixtures/external/vitest/error_and_passing.xml create mode 100644 tests/fixtures/external/vitest/plain_error.xml create mode 100644 tests/fixtures/external/vitest/reference_error_thrown.xml create mode 100644 tests/fixtures/external/vitest/syntax_error_thrown.xml create mode 100644 tests/fixtures/external/vitest/type_error_thrown.xml diff --git a/tests/fixtures/external/record_vitest.py b/tests/fixtures/external/record_vitest.py index b60006955..2d5a060a6 100644 --- a/tests/fixtures/external/record_vitest.py +++ b/tests/fixtures/external/record_vitest.py @@ -66,6 +66,31 @@ + "test('reads', () => {\n" + " throw new Error('the _read() method is not implemented');\n});\n", ), + "plain_error": ( + "j.test.ts", + HEADER + + "test('loads', () => {\n" + + " throw new Error('Dataset \"usage\" is not a report dataset');\n});\n", + ), + "type_error_thrown": ( + "k.test.ts", + HEADER + "test('calls', () => {\n const load: any = undefined;\n load();\n});\n", + ), + "reference_error_thrown": ( + "l.test.ts", + HEADER + "test('uses', () => {\n // @ts-ignore\n load();\n});\n", + ), + "syntax_error_thrown": ( + "m.test.ts", + HEADER + "test('parses', () => {\n JSON.parse('{');\n});\n", + ), + "error_and_passing": ( + "n.test.ts", + HEADER + + "test('loads', () => {\n" + + " throw new Error('Dataset \"usage\" is not a report dataset');\n});\n" + + "test('holds', () => {\n expect(1).toBe(1);\n});\n", + ), "syntax_error": ( "f.test.ts", HEADER + "test('parses', () => {\n expect(1 +).toBe(1);\n});\n", diff --git a/tests/fixtures/external/vitest/error_and_passing.xml b/tests/fixtures/external/vitest/error_and_passing.xml new file mode 100644 index 000000000..5de77e079 --- /dev/null +++ b/tests/fixtures/external/vitest/error_and_passing.xml @@ -0,0 +1,15 @@ + + + + + + +Error: Dataset "usage" is not a report dataset + ❯ n.test.ts:3:9 + + + + + + diff --git a/tests/fixtures/external/vitest/plain_error.xml b/tests/fixtures/external/vitest/plain_error.xml new file mode 100644 index 000000000..5e6bbe18d --- /dev/null +++ b/tests/fixtures/external/vitest/plain_error.xml @@ -0,0 +1,13 @@ + + + + + + +Error: Dataset "usage" is not a report dataset + ❯ j.test.ts:3:9 + + + + diff --git a/tests/fixtures/external/vitest/reference_error_thrown.xml b/tests/fixtures/external/vitest/reference_error_thrown.xml new file mode 100644 index 000000000..86a703cfd --- /dev/null +++ b/tests/fixtures/external/vitest/reference_error_thrown.xml @@ -0,0 +1,13 @@ + + + + + + +ReferenceError: load is not defined + ❯ l.test.ts:3:9 + + + + diff --git a/tests/fixtures/external/vitest/syntax_error_thrown.xml b/tests/fixtures/external/vitest/syntax_error_thrown.xml new file mode 100644 index 000000000..ea710bd91 --- /dev/null +++ b/tests/fixtures/external/vitest/syntax_error_thrown.xml @@ -0,0 +1,13 @@ + + + + + + +SyntaxError: Unexpected end of JSON input + ❯ m.test.ts:3:9 + + + + diff --git a/tests/fixtures/external/vitest/type_error_thrown.xml b/tests/fixtures/external/vitest/type_error_thrown.xml new file mode 100644 index 000000000..2b1bcd714 --- /dev/null +++ b/tests/fixtures/external/vitest/type_error_thrown.xml @@ -0,0 +1,13 @@ + + + + + + +TypeError: load is not a function + ❯ k.test.ts:3:9 + + + + diff --git a/tests/profiles/test_node_npm_interpret_red.py b/tests/profiles/test_node_npm_interpret_red.py index 26a8d4ae5..d2cdeb2ef 100644 --- a/tests/profiles/test_node_npm_interpret_red.py +++ b/tests/profiles/test_node_npm_interpret_red.py @@ -70,6 +70,51 @@ def test_a_file_that_did_not_load_a_passing_test_or_an_empty_run_is_rejected_wit assert any(reason in problem for problem in problems), problems +def test_a_test_failing_with_a_plain_error_is_red(profile, project: Project) -> None: + ok, problems = judge(profile, project, "plain_error", ["j.test.ts"]) + + assert ok, problems + assert problems == [] + + +def test_a_test_failing_with_a_plain_error_is_red_in_the_stubbed_reading_too( + profile, project: Project +) -> None: + done = project.run(profile.red_command(["j.test.ts"]), recording="plain_error") + + ok, problems = profile.interpret_red(done.stdout + done.stderr, done.returncode, stubbed=True) + + assert ok, problems + + +@pytest.mark.parametrize( + ("recording", "file", "error_type"), + [ + ("type_error_thrown", "k.test.ts", "TypeError"), + ("reference_error_thrown", "l.test.ts", "ReferenceError"), + ("syntax_error_thrown", "m.test.ts", "SyntaxError"), + ], + ids=["a type error", "a reference error", "a syntax error in the test body"], +) +def test_a_test_failing_with_a_missing_name_or_a_broken_test_is_rejected_naming_the_type( + profile, project: Project, recording: str, file: str, error_type: str +) -> None: + ok, problems = judge(profile, project, recording, [file]) + + assert not ok + assert any(error_type in problem for problem in problems), problems + + +def test_a_plain_error_failure_beside_a_passing_test_is_rejected_for_the_passing_test( + profile, project: Project +) -> None: + ok, problems = judge(profile, project, "error_and_passing", ["n.test.ts"]) + + assert not ok + assert any("passed" in problem for problem in problems), problems + assert not any("recognise" in problem for problem in problems), problems + + @pytest.mark.parametrize( ("recording", "file"), [("missing_import", "c.test.ts"), ("syntax_error", "f.test.ts")], From 1ec91589d45c1532853f4d2fc8cb8c0d68d25278 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Sat, 14 Nov 2026 08:42:20 -0700 Subject: [PATCH 2/5] feat: A failing test with a plain error is accepted as red With this in, the tests of the previous commit pass. The Node profile judges a tests commit red only when a failing test's JUnit `type` is `AssertionError`, or its message is exactly `not implemented`. Unit: js-red-check-accepts-a-thrown-error/1 --- CHANGELOG.md | 4 ++++ docs/toolchain-profiles.md | 4 ++-- src/agent_build_kit/profiles/node_npm.py | 19 +++++++++++++++---- tests/profiles/test_node_npm_interpret_red.py | 2 -- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c83a75c1..ae1039099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ deselected (pytest exit 5), never ran again, and a flake in it failed the unit; the rerun now takes the same by-id selection as the repeat check, and a marker expression of an e2e run is still kept. +- The JavaScript red check now accepts a test that fails with a plain `Error` thrown by the + code under test, as it does an assertion failure. A `TypeError`, `ReferenceError` or + `SyntaxError` is still refused, and the reason names the type, since it means a name is + missing or the test is broken. - The GitHub issues tracker now lists the issues changed since a cursor (pull requests left out, unchanged pages not transferred again), links a pull request with a `Part of #` line diff --git a/docs/toolchain-profiles.md b/docs/toolchain-profiles.md index 55226d388..58850a398 100644 --- a/docs/toolchain-profiles.md +++ b/docs/toolchain-profiles.md @@ -202,8 +202,8 @@ And what it knows: - `runs_path`: the file's own name has `.test` or `.spec` before the extension; Vitest reports no suite for a helper or data file under `__tests__`, so those are not given to the runner. - `interpret_red` judges only the suites named after the given files, which the red - command printed before the report: a failure of type `AssertionError` or an error whose - message is `not implemented` is red; a file that failed to load (its testcase is named as + command printed before the report: a failure of type `AssertionError` or a plain `Error` is red, + while a `TypeError`, `ReferenceError` or `SyntaxError` is rejected as a missing name or a broken test; a file that failed to load (its testcase is named as the file), any passing test, a given file with no failing test and any other failure are each rejected with a reason. - `stub_violations` (parsed with tree-sitter): every function, method or arrow body, at any diff --git a/src/agent_build_kit/profiles/node_npm.py b/src/agent_build_kit/profiles/node_npm.py index f3b72e3d0..b904a27fc 100644 --- a/src/agent_build_kit/profiles/node_npm.py +++ b/src/agent_build_kit/profiles/node_npm.py @@ -231,6 +231,10 @@ def _chosen(folder: Path, check: tuple[str, list[str]]) -> list[str]: # can tell their suites from any other the runner's substring filters selected. _RED_FILE = "abk red file: " _NOT_IMPLEMENTED = "not implemented" +# Failure types that are red: an assertion, or an error thrown by the code under test. +_RED_ERRORS = ("AssertionError", "Error") +# Failure types that say a name is missing or the test is broken, never missing behaviour. +_BROKEN_TEST_ERRORS = ("TypeError", "ReferenceError", "SyntaxError") # What the profile's own commands create in a project: type-check build info and the Vite and # Vitest caches. _CREATED = ["**/*.tsbuildinfo", "node_modules/.vite", "node_modules/.vitest", ".vite"] @@ -697,9 +701,10 @@ def interpret_red( ) -> tuple[bool, list[str]]: """Red for the right reason, judged on the suites of the files the red command was given (it lists them before the report): every test that ran failed by an assertion - or by an error saying `not implemented`. A file that failed to load (an unresolved - import, a syntax error), a passing test and a run with no suite of those files are - rejected, each with its reason. This is also the stubbed reading (`stubbed`): a file + or by a plain `Error` (a `TypeError`, `ReferenceError` or `SyntaxError` is a broken + test). A file that failed to load (an unresolved import, a syntax error), a passing + test and a run with no suite of those files are rejected, each with its reason. + This is also the stubbed reading (`stubbed`): a file that failed to load is rejected either way.""" _, console = split_report(output) given = { @@ -728,7 +733,13 @@ def interpret_red( f"{name}: the test file could not be loaded, so no test failed for " f"the missing behaviour ({first})" ) - elif bad.get("type") != "AssertionError" and message != _NOT_IMPLEMENTED: + elif bad.get("type") in _BROKEN_TEST_ERRORS: + first = message.split("\n", 1)[0] + problems.append( + f"{name}: failed with a {bad.get('type')}: a missing name or a broken " + f"test, not missing behaviour ({first})" + ) + elif bad.get("type") not in _RED_ERRORS: problems.append(f"{name}: {UNRECOGNISED_FAILURE}") if passed: problems.append(passed_problem(passed)) diff --git a/tests/profiles/test_node_npm_interpret_red.py b/tests/profiles/test_node_npm_interpret_red.py index d2cdeb2ef..b8f828ea5 100644 --- a/tests/profiles/test_node_npm_interpret_red.py +++ b/tests/profiles/test_node_npm_interpret_red.py @@ -51,14 +51,12 @@ def test_a_test_failing_by_assertion_or_by_not_implemented_is_red( ("syntax_error", "f.test.ts", "could not be loaded"), ("passing", "d.test.ts", "passed"), ("empty", "a.test.ts", "nothing failed"), - ("other_not_implemented", "i.test.ts", "not one we recognise"), ], ids=[ "an unresolved import", "a syntax error", "a passing test", "an empty run", - "an error that only mentions not implemented", ], ) def test_a_file_that_did_not_load_a_passing_test_or_an_empty_run_is_rejected_with_a_reason( From 294862a20305432ee85927eec338ce710e4a3fe5 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Mon, 16 Nov 2026 09:42:17 -0700 Subject: [PATCH 3/5] changelog: separate bullets joined by the union merge --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae1039099..a593cedbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ deselected (pytest exit 5), never ran again, and a flake in it failed the unit; the rerun now takes the same by-id selection as the repeat check, and a marker expression of an e2e run is still kept. + - The JavaScript red check now accepts a test that fails with a plain `Error` thrown by the code under test, as it does an assertion failure. A `TypeError`, `ReferenceError` or `SyntaxError` is still refused, and the reason names the type, since it means a name is From c2ccfc82c5de8f277bdf177792eb0f24aad9b012 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Mon, 16 Nov 2026 15:58:09 -0700 Subject: [PATCH 4/5] fix: The Node red check accepts a failing test with a plain Error Unit: js-red-check-accepts-a-thrown-error/1 --- tests/replay/test_replay_unit_build_tier.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/replay/test_replay_unit_build_tier.py b/tests/replay/test_replay_unit_build_tier.py index b13624597..5ced668e3 100644 --- a/tests/replay/test_replay_unit_build_tier.py +++ b/tests/replay/test_replay_unit_build_tier.py @@ -101,4 +101,6 @@ def test_the_build_without_the_drivers_environment_is_skipped_naming_what_is_mis assert "1 skipped" in output, output assert build.CASSETTES in output and build.MARKER in output, output - assert "error" not in output.lower(), output + skip_lines = [line for line in output.splitlines() if line.startswith("SKIPPED")] + assert skip_lines, output + assert not any("error" in line.lower() for line in skip_lines), output From 5bf7fd9f6ebb631f7f80b54bff2ac5f05dfb50d4 Mon Sep 17 00:00:00 2001 From: Ben Matson Date: Mon, 16 Nov 2026 17:00:24 -0700 Subject: [PATCH 5/5] fix: The Node red check accepts a failing test with a plain Error (review round 1) Unit: js-red-check-accepts-a-thrown-error/1 --- tests/replay/test_replay_unit_build_tier.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/replay/test_replay_unit_build_tier.py b/tests/replay/test_replay_unit_build_tier.py index 5ced668e3..a2c661251 100644 --- a/tests/replay/test_replay_unit_build_tier.py +++ b/tests/replay/test_replay_unit_build_tier.py @@ -104,3 +104,4 @@ def test_the_build_without_the_drivers_environment_is_skipped_naming_what_is_mis skip_lines = [line for line in output.splitlines() if line.startswith("SKIPPED")] assert skip_lines, output assert not any("error" in line.lower() for line in skip_lines), output + assert done.returncode == 0, output