diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs index 33c1b757bbf..3d79125f6a9 100644 --- a/test/wpt/runner/runner.mjs +++ b/test/wpt/runner/runner.mjs @@ -218,7 +218,7 @@ export class WPTRunner extends EventEmitter { if (message.type === 'result') { this.handleIndividualTestCompletion(message, status, test, meta, result) } else if (message.type === 'completion') { - this.handleTestCompletion(worker) + this.handleTestCompletion(worker, status, test) } else if (message.type === 'error') { this.#uncaughtExceptions.push({ error: message.error, test }) this.#stats.failedTests += 1 @@ -318,8 +318,36 @@ export class WPTRunner extends EventEmitter { * Called after all the tests in a worker are completed. * @param {Worker} worker */ - handleTestCompletion (worker) { + handleTestCompletion (worker, status, path) { worker.terminate() + + const { file } = status + const hasExpectedFailures = !!file.fail + const testHasFailures = !!this.#statusOutput?.[path] + const failed = this.#statusOutput?.[path] ?? [] + + if (hasExpectedFailures !== testHasFailures) { + console.log({ expected: file.fail, failed }) + + if (failed.length === 0) { + console.log(colors('Tests are marked as failure but did not fail, yay!', 'red')) + } else if (!hasExpectedFailures) { + console.log(colors('Test failed but there were no expected errors.', 'red')) + } + + process.exitCode = 1 + } else if (hasExpectedFailures && testHasFailures) { + const diff = [ + ...file.fail.filter(x => !failed.includes(x)), + ...failed.filter(x => !file.fail.includes(x)) + ] + + if (diff.length) { + console.log({ diff }) + console.log(colors('Expected failures did not match actual failures', 'red')) + process.exitCode = 1 + } + } } /** diff --git a/test/wpt/runner/util.mjs b/test/wpt/runner/util.mjs index ec284df68b2..6bf8b777123 100644 --- a/test/wpt/runner/util.mjs +++ b/test/wpt/runner/util.mjs @@ -168,5 +168,5 @@ export function resolveStatusPath (path, status) { } } - return { topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} } + return { fullPath: path, topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} } } diff --git a/test/wpt/status/FileAPI.status.json b/test/wpt/status/FileAPI.status.json index c64d255a0d3..da7aa96c0c8 100644 --- a/test/wpt/status/FileAPI.status.json +++ b/test/wpt/status/FileAPI.status.json @@ -11,7 +11,8 @@ "skip": true }, "Blob-stream.any.js": { - "fail": [ + "note": "fails in node v18", + "flaky": [ "Reading Blob.stream() with BYOB reader" ] } diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json index c94ef2f1c3b..9d63c1df11c 100644 --- a/test/wpt/status/fetch.status.json +++ b/test/wpt/status/fetch.status.json @@ -3,10 +3,7 @@ "abort": { "general.any.js": { "fail": [ - "Already aborted signal rejects immediately", - "Underlying connection is closed when aborting after receiving response - no-cors", - "Stream errors once aborted. Underlying connection closed.", - "Readable stream synchronously cancels with AbortError if aborted before reading" + "Already aborted signal rejects immediately" ] }, "cache.https.any.js": { @@ -52,9 +49,6 @@ }, "referrer.any.js": { "fail": [ - "origin-when-cross-origin policy on a cross-origin URL", - "origin-when-cross-origin policy on a cross-origin URL after same-origin redirection", - "origin-when-cross-origin policy on a same-origin URL after cross-origin redirection", "origin-when-cross-origin policy on a same-origin URL" ] }, @@ -100,7 +94,7 @@ }, "stream-safe-creation.any.js": { "note": "tests are very finnicky", - "fail": [ + "flaky": [ "throwing Object.prototype.type accessor should not affect stream creation by 'fetch'", "Object.prototype.type accessor returning invalid value should not affect stream creation by 'fetch'", "throwing Object.prototype.highWaterMark accessor should not affect stream creation by 'fetch'", @@ -344,7 +338,6 @@ "Adding invalid request header \"Accept-Encoding: KO\"", "Adding invalid request header \"Access-Control-Request-Headers: KO\"", "Adding invalid request header \"Access-Control-Request-Method: KO\"", - "Adding invalid request header \"Access-Control-Request-Private-Network: KO\"", "Adding invalid request header \"Connection: KO\"", "Adding invalid request header \"Content-Length: KO\"", "Adding invalid request header \"Cookie: KO\"", @@ -394,7 +387,8 @@ ] }, "response-consume-stream.any.js": { - "fail": [ + "note": "only fail in node v18", + "flaky": [ "Read blob response's body as readableStream with mode=byob", "Read text response's body as readableStream with mode=byob", "Read URLSearchParams response's body as readableStream with mode=byob", diff --git a/test/wpt/status/websockets.status.json b/test/wpt/status/websockets.status.json index 9f5a6308a1d..edd27727820 100644 --- a/test/wpt/status/websockets.status.json +++ b/test/wpt/status/websockets.status.json @@ -16,12 +16,6 @@ } } }, - "Create-blocked-port.any.js": { - "note": "TODO(@KhafraDev): investigate failure", - "fail": [ - "Basic check" - ] - }, "Send-binary-arraybufferview-float32.any.js": { "note": "TODO(@KhafraDev): investigate failure", "fail": [ @@ -58,12 +52,6 @@ "Send binary data on a WebSocket - ArrayBufferView - Uint32Array with offset - Connection should be closed" ] }, - "basic-auth.any.js": { - "note": "TODO(@KhafraDev): investigate failure", - "fail": [ - "HTTP basic authentication should work with WebSockets" - ] - }, "Create-on-worker-shutdown.any.js": { "skip": true, "//": "Node.js workers are different from web workers & don't work with blob: urls"