Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@
takes the same by-id selection as the repeat check, and a marker expression of an e2e run is
still kept.

- The JavaScript red check now accepts a test that fails with a plain `Error` thrown by the
code under test, as it does an assertion failure. A `TypeError`, `ReferenceError` or
`SyntaxError` is still refused, and the reason names the type, since it means a name is
missing or the test is broken.

- The GitHub issues tracker now lists the issues changed since a cursor (pull requests left
out, unchanged pages not transferred again), links a pull request with a `Part of #<n>` line
that does not close the issue, reaches a GitHub Enterprise Server host through the `host`
Expand Down
4 changes: 2 additions & 2 deletions docs/toolchain-profiles.md
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,8 @@ And what it knows:
- `runs_path`: the file's own name has `.test` or `.spec` before the extension; Vitest reports
no suite for a helper or data file under `__tests__`, so those are not given to the runner.
- `interpret_red` judges only the suites named after the given files, which the red
command printed before the report: a failure of type `AssertionError` or an error whose
message is `not implemented` is red; a file that failed to load (its testcase is named as
command printed before the report: a failure of type `AssertionError` or a plain `Error` is red,
while a `TypeError`, `ReferenceError` or `SyntaxError` is rejected as a missing name or a broken test; a file that failed to load (its testcase is named as
the file), any passing test, a given file with no failing test and any other failure are
each rejected with a reason.
- `stub_violations` (parsed with tree-sitter): every function, method or arrow body, at any
Expand Down
19 changes: 15 additions & 4 deletions src/agent_build_kit/profiles/node_npm.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,10 @@ def _chosen(folder: Path, check: tuple[str, list[str]]) -> list[str]:
# can tell their suites from any other the runner's substring filters selected.
_RED_FILE = "abk red file: "
_NOT_IMPLEMENTED = "not implemented"
# Failure types that are red: an assertion, or an error thrown by the code under test.
_RED_ERRORS = ("AssertionError", "Error")
# Failure types that say a name is missing or the test is broken, never missing behaviour.
_BROKEN_TEST_ERRORS = ("TypeError", "ReferenceError", "SyntaxError")
# What the profile's own commands create in a project: type-check build info and the Vite and
# Vitest caches.
_CREATED = ["**/*.tsbuildinfo", "node_modules/.vite", "node_modules/.vitest", ".vite"]
Expand Down Expand Up @@ -697,9 +701,10 @@ def interpret_red(
) -> tuple[bool, list[str]]:
"""Red for the right reason, judged on the suites of the files the red command was
given (it lists them before the report): every test that ran failed by an assertion
or by an error saying `not implemented`. A file that failed to load (an unresolved
import, a syntax error), a passing test and a run with no suite of those files are
rejected, each with its reason. This is also the stubbed reading (`stubbed`): a file
or by a plain `Error` (a `TypeError`, `ReferenceError` or `SyntaxError` is a broken
test). A file that failed to load (an unresolved import, a syntax error), a passing
test and a run with no suite of those files are rejected, each with its reason.
This is also the stubbed reading (`stubbed`): a file
that failed to load is rejected either way."""
_, console = split_report(output)
given = {
Expand Down Expand Up @@ -728,7 +733,13 @@ def interpret_red(
f"{name}: the test file could not be loaded, so no test failed for "
f"the missing behaviour ({first})"
)
elif bad.get("type") != "AssertionError" and message != _NOT_IMPLEMENTED:
elif bad.get("type") in _BROKEN_TEST_ERRORS:
first = message.split("\n", 1)[0]
problems.append(
f"{name}: failed with a {bad.get('type')}: a missing name or a broken "
f"test, not missing behaviour ({first})"
)
elif bad.get("type") not in _RED_ERRORS:
problems.append(f"{name}: {UNRECOGNISED_FAILURE}")
if passed:
problems.append(passed_problem(passed))
Expand Down
25 changes: 25 additions & 0 deletions tests/fixtures/external/record_vitest.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,31 @@
+ "test('reads', () => {\n"
+ " throw new Error('the _read() method is not implemented');\n});\n",
),
"plain_error": (
"j.test.ts",
HEADER
+ "test('loads', () => {\n"
+ " throw new Error('Dataset \"usage\" is not a report dataset');\n});\n",
),
"type_error_thrown": (
"k.test.ts",
HEADER + "test('calls', () => {\n const load: any = undefined;\n load();\n});\n",
),
"reference_error_thrown": (
"l.test.ts",
HEADER + "test('uses', () => {\n // @ts-ignore\n load();\n});\n",
),
"syntax_error_thrown": (
"m.test.ts",
HEADER + "test('parses', () => {\n JSON.parse('{');\n});\n",
),
"error_and_passing": (
"n.test.ts",
HEADER
+ "test('loads', () => {\n"
+ " throw new Error('Dataset \"usage\" is not a report dataset');\n});\n"
+ "test('holds', () => {\n expect(1).toBe(1);\n});\n",
),
"syntax_error": (
"f.test.ts",
HEADER + "test('parses', () => {\n expect(1 +).toBe(1);\n});\n",
Expand Down
15 changes: 15 additions & 0 deletions tests/fixtures/external/vitest/error_and_passing.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<!-- tool: vitest 5.0.4 linux-x64 node-v22.23.2
command: vitest run --reporter=junit --passWithNoTests --outputFile=<path> -->
<?xml version="1.0" encoding="UTF-8" ?>
<testsuites name="vitest tests" tests="2" failures="1" errors="0" time="0">
<testsuite name="n.test.ts" timestamp="0" hostname="0" tests="2" failures="1" errors="0" skipped="0" time="0">
<testcase classname="n.test.ts" name="loads" time="0">
<failure message="Dataset &quot;usage&quot; is not a report dataset" type="Error">
Error: Dataset "usage" is not a report dataset
❯ n.test.ts:3:9
</failure>
</testcase>
<testcase classname="n.test.ts" name="holds" time="0">
</testcase>
</testsuite>
</testsuites>
13 changes: 13 additions & 0 deletions tests/fixtures/external/vitest/plain_error.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<!-- tool: vitest 5.0.4 linux-x64 node-v22.23.2
command: vitest run --reporter=junit --passWithNoTests --outputFile=<path> -->
<?xml version="1.0" encoding="UTF-8" ?>
<testsuites name="vitest tests" tests="1" failures="1" errors="0" time="0">
<testsuite name="j.test.ts" timestamp="0" hostname="0" tests="1" failures="1" errors="0" skipped="0" time="0">
<testcase classname="j.test.ts" name="loads" time="0">
<failure message="Dataset &quot;usage&quot; is not a report dataset" type="Error">
Error: Dataset "usage" is not a report dataset
❯ j.test.ts:3:9
</failure>
</testcase>
</testsuite>
</testsuites>
13 changes: 13 additions & 0 deletions tests/fixtures/external/vitest/reference_error_thrown.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<!-- tool: vitest 5.0.4 linux-x64 node-v22.23.2
command: vitest run --reporter=junit --passWithNoTests --outputFile=<path> -->
<?xml version="1.0" encoding="UTF-8" ?>
<testsuites name="vitest tests" tests="1" failures="1" errors="0" time="0">
<testsuite name="l.test.ts" timestamp="0" hostname="0" tests="1" failures="1" errors="0" skipped="0" time="0">
<testcase classname="l.test.ts" name="uses" time="0">
<failure message="load is not defined" type="ReferenceError">
ReferenceError: load is not defined
❯ l.test.ts:3:9
</failure>
</testcase>
</testsuite>
</testsuites>
13 changes: 13 additions & 0 deletions tests/fixtures/external/vitest/syntax_error_thrown.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<!-- tool: vitest 5.0.4 linux-x64 node-v22.23.2
command: vitest run --reporter=junit --passWithNoTests --outputFile=<path> -->
<?xml version="1.0" encoding="UTF-8" ?>
<testsuites name="vitest tests" tests="1" failures="1" errors="0" time="0">
<testsuite name="m.test.ts" timestamp="0" hostname="0" tests="1" failures="1" errors="0" skipped="0" time="0">
<testcase classname="m.test.ts" name="parses" time="0">
<failure message="Unexpected end of JSON input" type="SyntaxError">
SyntaxError: Unexpected end of JSON input
❯ m.test.ts:3:9
</failure>
</testcase>
</testsuite>
</testsuites>
13 changes: 13 additions & 0 deletions tests/fixtures/external/vitest/type_error_thrown.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<!-- tool: vitest 5.0.4 linux-x64 node-v22.23.2
command: vitest run --reporter=junit --passWithNoTests --outputFile=<path> -->
<?xml version="1.0" encoding="UTF-8" ?>
<testsuites name="vitest tests" tests="1" failures="1" errors="0" time="0">
<testsuite name="k.test.ts" timestamp="0" hostname="0" tests="1" failures="1" errors="0" skipped="0" time="0">
<testcase classname="k.test.ts" name="calls" time="0">
<failure message="load is not a function" type="TypeError">
TypeError: load is not a function
❯ k.test.ts:3:9
</failure>
</testcase>
</testsuite>
</testsuites>
47 changes: 45 additions & 2 deletions tests/profiles/test_node_npm_interpret_red.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,12 @@ def test_a_test_failing_by_assertion_or_by_not_implemented_is_red(
("syntax_error", "f.test.ts", "could not be loaded"),
("passing", "d.test.ts", "passed"),
("empty", "a.test.ts", "nothing failed"),
("other_not_implemented", "i.test.ts", "not one we recognise"),
],
ids=[
"an unresolved import",
"a syntax error",
"a passing test",
"an empty run",
"an error that only mentions not implemented",
],
)
def test_a_file_that_did_not_load_a_passing_test_or_an_empty_run_is_rejected_with_a_reason(
Expand All @@ -70,6 +68,51 @@ def test_a_file_that_did_not_load_a_passing_test_or_an_empty_run_is_rejected_wit
assert any(reason in problem for problem in problems), problems


def test_a_test_failing_with_a_plain_error_is_red(profile, project: Project) -> None:
ok, problems = judge(profile, project, "plain_error", ["j.test.ts"])

assert ok, problems
assert problems == []


def test_a_test_failing_with_a_plain_error_is_red_in_the_stubbed_reading_too(
profile, project: Project
) -> None:
done = project.run(profile.red_command(["j.test.ts"]), recording="plain_error")

ok, problems = profile.interpret_red(done.stdout + done.stderr, done.returncode, stubbed=True)

assert ok, problems


@pytest.mark.parametrize(
("recording", "file", "error_type"),
[
("type_error_thrown", "k.test.ts", "TypeError"),
("reference_error_thrown", "l.test.ts", "ReferenceError"),
("syntax_error_thrown", "m.test.ts", "SyntaxError"),
],
ids=["a type error", "a reference error", "a syntax error in the test body"],
)
def test_a_test_failing_with_a_missing_name_or_a_broken_test_is_rejected_naming_the_type(
profile, project: Project, recording: str, file: str, error_type: str
) -> None:
ok, problems = judge(profile, project, recording, [file])

assert not ok
assert any(error_type in problem for problem in problems), problems


def test_a_plain_error_failure_beside_a_passing_test_is_rejected_for_the_passing_test(
profile, project: Project
) -> None:
ok, problems = judge(profile, project, "error_and_passing", ["n.test.ts"])

assert not ok
assert any("passed" in problem for problem in problems), problems
assert not any("recognise" in problem for problem in problems), problems


@pytest.mark.parametrize(
("recording", "file"),
[("missing_import", "c.test.ts"), ("syntax_error", "f.test.ts")],
Expand Down
5 changes: 4 additions & 1 deletion tests/replay/test_replay_unit_build_tier.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,7 @@ def test_the_build_without_the_drivers_environment_is_skipped_naming_what_is_mis

assert "1 skipped" in output, output
assert build.CASSETTES in output and build.MARKER in output, output
assert "error" not in output.lower(), output
skip_lines = [line for line in output.splitlines() if line.startswith("SKIPPED")]
assert skip_lines, output
assert not any("error" in line.lower() for line in skip_lines), output
assert done.returncode == 0, output