From c009d6a89de6d0fa9fcfb53383c265b6f9725a61 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 6 Dec 2023 16:44:08 +0100 Subject: [PATCH 01/19] fix: more sensible stack trace from dump error (#2503) * fix: more sensible stack trace from dump error * Update lib/core/errors.js Co-authored-by: Carlos Fuentes --------- Co-authored-by: Carlos Fuentes --- lib/api/readable.js | 42 ++++++++++++++++++------------------------ lib/core/errors.js | 12 +++++++++++- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/lib/api/readable.js b/lib/api/readable.js index 356a03aedd9..c6e142b8ab9 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -4,7 +4,7 @@ const assert = require('assert') const { Readable } = require('stream') -const { RequestAbortedError, NotSupportedError, InvalidArgumentError } = require('../core/errors') +const { RequestAbortedError, NotSupportedError, InvalidArgumentError, AbortError } = require('../core/errors') const util = require('../core/util') const { ReadableStreamFrom, toUSVString } = require('../core/util') @@ -163,37 +163,31 @@ module.exports = class BodyReadable extends Readable { return this[kBody] } - dump (opts) { - let limit = opts && Number.isFinite(opts.limit) ? opts.limit : 262144 - const signal = opts && opts.signal - - if (signal) { - try { - if (typeof signal !== 'object' || !('aborted' in signal)) { - throw new InvalidArgumentError('signal must be an AbortSignal') - } - util.throwIfAborted(signal) - } catch (err) { - return Promise.reject(err) - } + async dump (opts) { + let limit = Number.isFinite(opts?.limit) ? opts.limit : 262144 + const signal = opts?.signal + + if (signal != null && (typeof signal !== 'object' || !('aborted' in signal))) { + throw new InvalidArgumentError('signal must be an AbortSignal') } + signal?.throwIfAborted() + if (this._readableState.closeEmitted) { - return Promise.resolve(null) + return null } - return new Promise((resolve, reject) => { - const signalListenerCleanup = signal - ? util.addAbortListener(signal, () => { - this.destroy() - }) - : noop + return await new Promise((resolve, reject) => { + const onAbort = () => { + this.destroy(signal.reason ?? new AbortError()) + } + signal?.addEventListener('abort', onAbort) this .on('close', function () { - signalListenerCleanup() - if (signal && signal.aborted) { - reject(signal.reason || Object.assign(new Error('The operation was aborted'), { name: 'AbortError' })) + signal?.removeEventListener('abort', onAbort) + if (signal?.aborted) { + reject(signal.reason ?? new AbortError()) } else { resolve(null) } diff --git a/lib/core/errors.js b/lib/core/errors.js index 7af704b462a..e9adb2064da 100644 --- a/lib/core/errors.js +++ b/lib/core/errors.js @@ -82,7 +82,16 @@ class InvalidReturnValueError extends UndiciError { } } -class RequestAbortedError extends UndiciError { +class AbortError extends UndiciError { + constructor (message) { + super(message) + Error.captureStackTrace(this, AbortError) + this.name = 'AbortError' + this.message = message || 'The operation was aborted' + } +} + +class RequestAbortedError extends AbortError { constructor (message) { super(message) Error.captureStackTrace(this, RequestAbortedError) @@ -207,6 +216,7 @@ class RequestRetryError extends UndiciError { } module.exports = { + AbortError, HTTPParserError, UndiciError, HeadersTimeoutError, From 84a99b06a7931ced23be0ccfa9bd3a63c9e079af Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 7 Dec 2023 06:59:42 +0100 Subject: [PATCH 02/19] refactor: remove some node compat (#2502) --- lib/api/readable.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/api/readable.js b/lib/api/readable.js index c6e142b8ab9..e6e71c4554c 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -46,11 +46,6 @@ module.exports = class BodyReadable extends Readable { } destroy (err) { - if (this.destroyed) { - // Node < 16 - return this - } - if (!err && !this._readableState.endEmitted) { err = new RequestAbortedError() } @@ -74,17 +69,6 @@ module.exports = class BodyReadable extends Readable { }) } - emit (ev, ...args) { - if (ev === 'data') { - // Node < 16.7 - this._readableState.dataEmitted = true - } else if (ev === 'error') { - // Node < 16 - this._readableState.errorEmitted = true - } - return super.emit(ev, ...args) - } - on (ev, ...args) { if (ev === 'data' || ev === 'readable') { this[kReading] = true From d41d58d2a9b8013a5d6b3bc8c8dfd94690af707c Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Thu, 7 Dec 2023 17:07:51 +0900 Subject: [PATCH 03/19] refactor: version cleanup (#2507) * refactor: version cleanup * remove throwIfAborted --- lib/core/util.js | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/lib/core/util.js b/lib/core/util.js index 5a28529b663..d221e98673b 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -334,19 +334,11 @@ function isDisturbed (body) { } function isErrored (body) { - return !!(body && ( - stream.isErrored - ? stream.isErrored(body) - : /state: 'errored'/.test(nodeUtil.inspect(body) - ))) + return !!(body && stream.isErrored(body)) } function isReadable (body) { - return !!(body && ( - stream.isReadable - ? stream.isReadable(body) - : /state: 'readable'/.test(nodeUtil.inspect(body) - ))) + return !!(body && stream.isReadable(body)) } function getSocketInfo (socket) { @@ -411,20 +403,6 @@ function isFormDataLike (object) { ) } -function throwIfAborted (signal) { - if (!signal) { return } - if (typeof signal.throwIfAborted === 'function') { - signal.throwIfAborted() - } else { - if (signal.aborted) { - // DOMException not available < v17.0.0 - const err = new Error('The operation was aborted') - err.name = 'AbortError' - throw err - } - } -} - function addAbortListener (signal, listener) { if ('addEventListener' in signal) { signal.addEventListener('abort', listener, { once: true }) @@ -495,7 +473,6 @@ module.exports = { getSocketInfo, isFormDataLike, buildURL, - throwIfAborted, addAbortListener, parseRangeHeader, nodeMajor, From de7dc7fabbbf7b97edb583b99003cb2222d4166f Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Fri, 8 Dec 2023 10:20:01 +0900 Subject: [PATCH 04/19] perf(fetch): Improve fetch of detaurl (#2479) * perf(fetch): Improve data url base64 * format * fix: comment position * add comment * add comment * suggestion change * perf: avoid replace * fixup * refactor * fixup * Revert "fixup" This reverts commit 058dc02d3509b19957379ad227c020df99bf7e3d. * fixup * remove --- lib/fetch/dataURL.js | 93 +++++++++++++++++++++++--------------------- lib/fetch/util.js | 21 ++++++---- 2 files changed, 62 insertions(+), 52 deletions(-) diff --git a/lib/fetch/dataURL.js b/lib/fetch/dataURL.js index 5d6b3940526..762017c8cc9 100644 --- a/lib/fetch/dataURL.js +++ b/lib/fetch/dataURL.js @@ -1,5 +1,4 @@ const assert = require('assert') -const { atob } = require('buffer') const { isomorphicDecode } = require('./util') const encoder = new TextEncoder() @@ -8,7 +7,8 @@ const encoder = new TextEncoder() * @see https://mimesniff.spec.whatwg.org/#http-token-code-point */ const HTTP_TOKEN_CODEPOINTS = /^[!#$%&'*+-.^_|~A-Za-z0-9]+$/ -const HTTP_WHITESPACE_REGEX = /(\u000A|\u000D|\u0009|\u0020)/ // eslint-disable-line +const HTTP_WHITESPACE_REGEX = /[\u000A|\u000D|\u0009|\u0020]/ // eslint-disable-line +const ASCII_WHITESPACE_REPLACE_REGEX = /[\u0009\u000A\u000C\u000D\u0020]/g // eslint-disable-line /** * @see https://mimesniff.spec.whatwg.org/#http-quoted-string-token-code-point */ @@ -188,20 +188,26 @@ function stringPercentDecode (input) { return percentDecode(bytes) } +function isHexCharByte (byte) { + // 0-9 A-F a-f + return (byte >= 0x30 && byte <= 0x39) || (byte >= 0x41 && byte <= 0x46) || (byte >= 0x61 && byte <= 0x66) +} + // https://url.spec.whatwg.org/#percent-decode /** @param {Uint8Array} input */ function percentDecode (input) { + const length = input.length // 1. Let output be an empty byte sequence. - /** @type {number[]} */ - const output = [] - + /** @type {Uint8Array} */ + const output = new Uint8Array(length) + let j = 0 // 2. For each byte byte in input: - for (let i = 0; i < input.length; i++) { + for (let i = 0; i < length; ++i) { const byte = input[i] // 1. If byte is not 0x25 (%), then append byte to output. if (byte !== 0x25) { - output.push(byte) + output[j++] = byte // 2. Otherwise, if byte is 0x25 (%) and the next two bytes // after byte in input are not in the ranges @@ -210,9 +216,9 @@ function percentDecode (input) { // to output. } else if ( byte === 0x25 && - !/^[0-9A-Fa-f]{2}$/i.test(String.fromCharCode(input[i + 1], input[i + 2])) + !(isHexCharByte(input[i + 1]) && isHexCharByte(input[i + 2])) ) { - output.push(0x25) + output[j++] = 0x25 // 3. Otherwise: } else { @@ -222,7 +228,7 @@ function percentDecode (input) { const bytePoint = Number.parseInt(nextTwoBytes, 16) // 2. Append a byte whose value is bytePoint to output. - output.push(bytePoint) + output[j++] = bytePoint // 3. Skip the next two bytes in input. i += 2 @@ -230,7 +236,7 @@ function percentDecode (input) { } // 3. Return output. - return Uint8Array.from(output) + return length === j ? output : output.subarray(0, j) } // https://mimesniff.spec.whatwg.org/#parse-a-mime-type @@ -410,19 +416,25 @@ function parseMIMEType (input) { /** @param {string} data */ function forgivingBase64 (data) { // 1. Remove all ASCII whitespace from data. - data = data.replace(/[\u0009\u000A\u000C\u000D\u0020]/g, '') // eslint-disable-line + data = data.replace(ASCII_WHITESPACE_REPLACE_REGEX, '') // eslint-disable-line + let dataLength = data.length // 2. If data’s code point length divides by 4 leaving // no remainder, then: - if (data.length % 4 === 0) { + if (dataLength % 4 === 0) { // 1. If data ends with one or two U+003D (=) code points, // then remove them from data. - data = data.replace(/=?=$/, '') + if (data.charCodeAt(dataLength - 1) === 0x003D) { + --dataLength + if (data.charCodeAt(dataLength - 1) === 0x003D) { + --dataLength + } + } } // 3. If data’s code point length divides by 4 leaving // a remainder of 1, then return failure. - if (data.length % 4 === 1) { + if (dataLength % 4 === 1) { return 'failure' } @@ -431,18 +443,12 @@ function forgivingBase64 (data) { // U+002F (/) // ASCII alphanumeric // then return failure. - if (/[^+/0-9A-Za-z]/.test(data)) { + if (/[^+/0-9A-Za-z]/.test(data.length === dataLength ? data : data.substring(0, dataLength))) { return 'failure' } - const binary = atob(data) - const bytes = new Uint8Array(binary.length) - - for (let byte = 0; byte < binary.length; byte++) { - bytes[byte] = binary.charCodeAt(byte) - } - - return bytes + const buffer = Buffer.from(data, 'base64') + return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) } // https://fetch.spec.whatwg.org/#collect-an-http-quoted-string @@ -570,55 +576,54 @@ function serializeAMimeType (mimeType) { /** * @see https://fetch.spec.whatwg.org/#http-whitespace - * @param {string} char + * @param {number} char */ function isHTTPWhiteSpace (char) { - return char === '\r' || char === '\n' || char === '\t' || char === ' ' + // "\r\n\t " + return char === 0x00d || char === 0x00a || char === 0x009 || char === 0x020 } /** * @see https://fetch.spec.whatwg.org/#http-whitespace * @param {string} str + * @param {boolean} [leading=true] + * @param {boolean} [trailing=true] */ function removeHTTPWhitespace (str, leading = true, trailing = true) { - let lead = 0 - let trail = str.length - 1 - + let i = 0; let j = str.length if (leading) { - for (; lead < str.length && isHTTPWhiteSpace(str[lead]); lead++); + while (j > i && isHTTPWhiteSpace(str.charCodeAt(i))) --i } - if (trailing) { - for (; trail > 0 && isHTTPWhiteSpace(str[trail]); trail--); + while (j > i && isHTTPWhiteSpace(str.charCodeAt(j - 1))) --j } - - return str.slice(lead, trail + 1) + return i === 0 && j === str.length ? str : str.substring(i, j) } /** * @see https://infra.spec.whatwg.org/#ascii-whitespace - * @param {string} char + * @param {number} char */ function isASCIIWhitespace (char) { - return char === '\r' || char === '\n' || char === '\t' || char === '\f' || char === ' ' + // "\r\n\t\f " + return char === 0x00d || char === 0x00a || char === 0x009 || char === 0x00c || char === 0x020 } /** * @see https://infra.spec.whatwg.org/#strip-leading-and-trailing-ascii-whitespace + * @param {string} str + * @param {boolean} [leading=true] + * @param {boolean} [trailing=true] */ function removeASCIIWhitespace (str, leading = true, trailing = true) { - let lead = 0 - let trail = str.length - 1 - + let i = 0; let j = str.length if (leading) { - for (; lead < str.length && isASCIIWhitespace(str[lead]); lead++); + while (j > i && isASCIIWhitespace(str.charCodeAt(i))) --i } - if (trailing) { - for (; trail > 0 && isASCIIWhitespace(str[trail]); trail--); + while (j > i && isASCIIWhitespace(str.charCodeAt(j - 1))) --j } - - return str.slice(lead, trail + 1) + return i === 0 && j === str.length ? str : str.substring(i, j) } module.exports = { diff --git a/lib/fetch/util.js b/lib/fetch/util.js index 96711267681..b729d70d1e8 100644 --- a/lib/fetch/util.js +++ b/lib/fetch/util.js @@ -897,22 +897,27 @@ function isReadableStreamLike (stream) { ) } -const MAXIMUM_ARGUMENT_LENGTH = 65535 - /** * @see https://infra.spec.whatwg.org/#isomorphic-decode - * @param {number[]|Uint8Array} input + * @param {Uint8Array} input */ function isomorphicDecode (input) { // 1. To isomorphic decode a byte sequence input, return a string whose code point // length is equal to input’s length and whose code points have the same values // as the values of input’s bytes, in the same order. - - if (input.length < MAXIMUM_ARGUMENT_LENGTH) { - return String.fromCharCode(...input) + const length = input.length + if ((2 << 15) - 1 > length) { + return String.fromCharCode.apply(null, input) } - - return input.reduce((previous, current) => previous + String.fromCharCode(current), '') + let result = ''; let i = 0 + let addition = (2 << 15) - 1 + while (i < length) { + if (i + addition > length) { + addition = length - i + } + result += String.fromCharCode.apply(null, input.subarray(i, i += addition)) + } + return result } /** From 7ab7d2c630f07b4aaa36637f95ae05b2a530036a Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Fri, 8 Dec 2023 09:03:09 +0100 Subject: [PATCH 05/19] feat: expose parseHeader (#2511) --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index 66d60785613..6898fb757d9 100644 --- a/index.js +++ b/index.js @@ -20,6 +20,7 @@ const { getGlobalDispatcher, setGlobalDispatcher } = require('./lib/global') const DecoratorHandler = require('./lib/handler/DecoratorHandler') const RedirectHandler = require('./lib/handler/RedirectHandler') const createRedirectInterceptor = require('./lib/interceptor/redirectInterceptor') +const { parseHeaders } = require('./lib/core/util') let hasCrypto try { @@ -45,6 +46,9 @@ module.exports.createRedirectInterceptor = createRedirectInterceptor module.exports.buildConnector = buildConnector module.exports.errors = errors +module.exports.util = { + parseHeaders +} function makeDispatcher (fn) { return (url, opts, handler) => { From 289c874eb8a41a462ffbb3e5751837f5d1ddc395 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:57:35 +0900 Subject: [PATCH 06/19] perf(fetch): optimize call `dispatch` (#2493) * perf(fetch): optimize call `dispatch` * fixup * format * fixup * refactor * fixup * fixup * fixup * perf * fix: add missing async * add comment * fix comment * fix comment * Revert "fix: add missing async" --- lib/fetch/index.js | 81 ++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 4471ee5d57c..b3b6e71eb79 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -9,7 +9,7 @@ const { filterResponse, makeResponse } = require('./response') -const { Headers } = require('./headers') +const { Headers, HeadersList } = require('./headers') const { Request, makeRequest } = require('./request') const zlib = require('zlib') const { @@ -2075,7 +2075,7 @@ async function httpNetworkFetch ( // 20. Return response. return response - async function dispatch ({ body }) { + function dispatch ({ body }) { const url = requestCurrentURL(request) /** @type {import('../..').Agent} */ const agent = fetchParams.controller.dispatcher @@ -2085,7 +2085,7 @@ async function httpNetworkFetch ( path: url.pathname + url.search, origin: url.origin, method: request.method, - body: fetchParams.controller.dispatcher.isMockActive ? request.body && (request.body.source || request.body.stream) : body, + body: agent.isMockActive ? request.body && (request.body.source || request.body.stream) : body, headers: request.headersList.entries, maxRedirections: 0, upgrade: request.mode === 'websocket' ? 'websocket' : undefined @@ -2106,59 +2106,57 @@ async function httpNetworkFetch ( } }, - onHeaders (status, headersList, resume, statusText) { + onHeaders (status, rawHeaders, resume, statusText) { if (status < 200) { return } + /** @type {string[]} */ let codings = [] let location = '' - const headers = new Headers() + const headersList = new HeadersList() - // For H2, the headers are a plain JS object + // For H2, the rawHeaders are a plain JS object // We distinguish between them and iterate accordingly - if (Array.isArray(headersList)) { - for (let n = 0; n < headersList.length; n += 2) { - const key = headersList[n + 0].toString('latin1') - const val = headersList[n + 1].toString('latin1') - if (key.toLowerCase() === 'content-encoding') { - // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 - // "All content-coding values are case-insensitive..." - codings = val.toLowerCase().split(',').map((x) => x.trim()) - } else if (key.toLowerCase() === 'location') { - location = val - } - - headers[kHeadersList].append(key, val) + if (Array.isArray(rawHeaders)) { + for (let i = 0; i < rawHeaders.length; i += 2) { + headersList.append(rawHeaders[i].toString('latin1'), rawHeaders[i + 1].toString('latin1')) } + const contentEncoding = headersList.get('content-encoding') + if (contentEncoding) { + // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 + // "All content-coding values are case-insensitive..." + codings = contentEncoding.toLowerCase().split(',').map((x) => x.trim()) + } + location = headersList.get('location') } else { - const keys = Object.keys(headersList) - for (const key of keys) { - const val = headersList[key] - if (key.toLowerCase() === 'content-encoding') { - // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 - // "All content-coding values are case-insensitive..." - codings = val.toLowerCase().split(',').map((x) => x.trim()).reverse() - } else if (key.toLowerCase() === 'location') { - location = val - } - - headers[kHeadersList].append(key, val) + const keys = Object.keys(rawHeaders) + for (let i = 0; i < keys.length; ++i) { + headersList.append(keys[i], rawHeaders[keys[i]]) + } + // For H2, The header names are already in lowercase, + // so we can avoid the `HeadersList#get` call here. + const contentEncoding = rawHeaders['content-encoding'] + if (contentEncoding) { + // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 + // "All content-coding values are case-insensitive..." + codings = contentEncoding.toLowerCase().split(',').map((x) => x.trim()).reverse() } + location = rawHeaders.location } this.body = new Readable({ read: resume }) const decoders = [] - const willFollow = request.redirect === 'follow' && - location && + const willFollow = location && request.redirect === 'follow' && redirectStatusSet.has(status) // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding if (request.method !== 'HEAD' && request.method !== 'CONNECT' && !nullBodyStatus.includes(status) && !willFollow) { - for (const coding of codings) { + for (let i = 0; i < codings.length; ++i) { + const coding = codings[i] // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.2 if (coding === 'x-gzip' || coding === 'gzip') { decoders.push(zlib.createGunzip({ @@ -2183,7 +2181,7 @@ async function httpNetworkFetch ( resolve({ status, statusText, - headersList: headers[kHeadersList], + headersList, body: decoders.length ? pipeline(this.body, ...decoders, () => { }) : this.body.on('error', () => {}) @@ -2237,24 +2235,21 @@ async function httpNetworkFetch ( reject(error) }, - onUpgrade (status, headersList, socket) { + onUpgrade (status, rawHeaders, socket) { if (status !== 101) { return } - const headers = new Headers() - - for (let n = 0; n < headersList.length; n += 2) { - const key = headersList[n + 0].toString('latin1') - const val = headersList[n + 1].toString('latin1') + const headersList = new HeadersList() - headers[kHeadersList].append(key, val) + for (let i = 0; i < rawHeaders.length; i += 2) { + headersList.append(rawHeaders[i].toString('latin1'), rawHeaders[i + 1].toString('latin1')) } resolve({ status, statusText: STATUS_CODES[status], - headersList: headers[kHeadersList], + headersList, socket }) From 0690568fca769d304cefada6663cedcb5b9c31c9 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:58:21 +0900 Subject: [PATCH 07/19] perf(util/parseHeaders): If the header name is buffer (#2501) * initial implementation * test: fix * compatible API * fix: tree * add benchmark * fix: lint * fix: benchmark * perf * use number key * remove unsafe * format & add comment * fix: benchmark import path * better benchmark * better benchmark * perf: rewrite tree * test: fuzz test * fix test * test * test: remove tree * refactor * refactor * suggested change * test: refactor * add use strict * test: refactor * add type comment * check length * test: perf * improve type * fix: type --- benchmarks/parseHeaders.mjs | 105 +++++++++++++++++++++++++++++ lib/core/constants.js | 2 + lib/core/tree.js | 129 ++++++++++++++++++++++++++++++++++++ lib/core/util.js | 32 ++++++--- package.json | 1 + test/tree.js | 40 +++++++++++ test/util.js | 5 +- 7 files changed, 303 insertions(+), 11 deletions(-) create mode 100644 benchmarks/parseHeaders.mjs create mode 100644 lib/core/tree.js create mode 100644 test/tree.js diff --git a/benchmarks/parseHeaders.mjs b/benchmarks/parseHeaders.mjs new file mode 100644 index 00000000000..6fb898062b3 --- /dev/null +++ b/benchmarks/parseHeaders.mjs @@ -0,0 +1,105 @@ +import { bench, group, run } from 'mitata' +import { parseHeaders } from '../lib/core/util.js' + +const target = [ + { + 'Content-Type': 'application/json', + Date: 'Wed, 01 Nov 2023 00:00:00 GMT', + 'Powered-By': 'NodeJS', + 'Content-Encoding': 'gzip', + 'Set-Cookie': '__Secure-ID=123; Secure; Domain=example.com', + 'Content-Length': '150', + Vary: 'Accept-Encoding, Accept, X-Requested-With' + }, + { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': '1234', + Date: 'Wed, 06 Dec 2023 12:47:57 GMT', + Server: 'Bing' + }, + { + 'Content-Type': 'image/jpeg', + 'Content-Length': '56789', + Date: 'Wed, 06 Dec 2023 12:48:12 GMT', + Server: 'Bing', + ETag: '"a1b2c3d4e5f6g7h8i9j0"' + }, + { + Cookie: 'session_id=1234567890abcdef', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + Host: 'www.bing.com', + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br' + }, + { + Location: 'https://www.bing.com/search?q=bing', + Status: '302 Found', + Date: 'Wed, 06 Dec 2023 12:48:27 GMT', + Server: 'Bing', + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': '0' + }, + { + 'Content-Type': + 'multipart/form-data; boundary=----WebKitFormBoundary1234567890', + 'Content-Length': '98765', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + Host: 'www.bing.com', + Accept: '*/*', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br' + }, + { + 'Content-Type': 'application/json; charset=UTF-8', + 'Content-Length': '2345', + Date: 'Wed, 06 Dec 2023 12:48:42 GMT', + Server: 'Bing', + Status: '200 OK', + 'Cache-Control': 'no-cache, no-store, must-revalidate' + }, + { + Host: 'www.example.com', + Connection: 'keep-alive', + Accept: 'text/html, application/xhtml+xml, application/xml;q=0.9,;q=0.8', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' + } +] + +const headers = Array.from(target, (x) => + Object.entries(x) + .flat() + .map((c) => Buffer.from(c)) +) + +const headersIrregular = Array.from( + target, + (x) => Object.entries(x) + .flat() + .map((c) => Buffer.from(c.toUpperCase())) +) + +// avoid JIT bias +bench('noop', () => {}) +bench('noop', () => {}) +bench('noop', () => {}) +bench('noop', () => {}) +bench('noop', () => {}) +bench('noop', () => {}) + +group('parseHeaders', () => { + bench('parseHeaders', () => { + for (let i = 0; i < headers.length; ++i) { + parseHeaders(headers[i]) + } + }) + bench('parseHeaders (irregular)', () => { + for (let i = 0; i < headersIrregular.length; ++i) { + parseHeaders(headersIrregular[i]) + } + }) +}) + +await new Promise((resolve) => setTimeout(resolve, 7000)) + +await run() diff --git a/lib/core/constants.js b/lib/core/constants.js index 0f827cc4ae0..6ec770dd533 100644 --- a/lib/core/constants.js +++ b/lib/core/constants.js @@ -1,3 +1,5 @@ +'use strict' + /** @type {Record} */ const headerNameLowerCasedRecord = {} diff --git a/lib/core/tree.js b/lib/core/tree.js new file mode 100644 index 00000000000..aa1641d217f --- /dev/null +++ b/lib/core/tree.js @@ -0,0 +1,129 @@ +'use strict' + +const { wellknownHeaderNames } = require('./constants') + +class TstNode { + /** @type {any} */ + value = null + /** @type {null | TstNode} */ + left = null + /** @type {null | TstNode} */ + middle = null + /** @type {null | TstNode} */ + right = null + /** @type {number} */ + code + /** + * @param {Uint8Array} key + * @param {any} value + */ + constructor (key, value) { + if (key.length === 0) { + throw new TypeError('Unreachable') + } + this.code = key[0] + if (key.length > 1) { + this.middle = new TstNode(key.subarray(1), value) + } else { + this.value = value + } + } + + /** + * @param {Uint8Array} key + * @param {any} value + */ + add (key, value) { + if (key.length === 0) { + throw new TypeError('Unreachable') + } + const code = key[0] + if (this.code === code) { + if (key.length === 1) { + this.value = value + } else if (this.middle !== null) { + this.middle.add(key.subarray(1), value) + } else { + this.middle = new TstNode(key.subarray(1), value) + } + } else if (this.code < code) { + if (this.left !== null) { + this.left.add(key, value) + } else { + this.left = new TstNode(key, value) + } + } else { + if (this.right !== null) { + this.right.add(key, value) + } else { + this.right = new TstNode(key, value) + } + } + } + + /** + * @param {Uint8Array} key + * @return {TstNode | null} + */ + search (key) { + const keylength = key.length + let index = 0 + let node = this + while (node !== null && index < keylength) { + let code = key[index] + // A-Z + if (code >= 0x41 && code <= 0x5a) { + // Lowercase for uppercase. + code |= 32 + } + while (node !== null) { + if (code === node.code) { + if (keylength === ++index) { + // Returns Node since it is the last key. + return node + } + node = node.middle + break + } + node = node.code < code ? node.left : node.right + } + } + return null + } +} + +class TernarySearchTree { + /** @type {TstNode | null} */ + node = null + + /** + * @param {Uint8Array} key + * @param {any} value + * */ + insert (key, value) { + if (this.node === null) { + this.node = new TstNode(key, value) + } else { + this.node.add(key, value) + } + } + + /** + * @param {Uint8Array} key + */ + lookup (key) { + return this.node?.search(key)?.value ?? null + } +} + +const tree = new TernarySearchTree() + +for (let i = 0; i < wellknownHeaderNames.length; ++i) { + const key = wellknownHeaderNames[i].toLowerCase() + tree.insert(Buffer.from(key), key) +} + +module.exports = { + TernarySearchTree, + tree +} diff --git a/lib/core/util.js b/lib/core/util.js index d221e98673b..75d31888221 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -10,6 +10,7 @@ const { Blob } = require('buffer') const nodeUtil = require('util') const { stringify } = require('querystring') const { headerNameLowerCasedRecord } = require('./constants') +const { tree } = require('./tree') const [nodeMajor, nodeMinor] = process.versions.node.split('.').map(v => Number(v)) @@ -219,26 +220,40 @@ function parseKeepAliveTimeout (val) { return m ? parseInt(m[1], 10) * 1000 : null } -function parseHeaders (headers, obj = {}) { +/** + * @param {string | Buffer} value + */ +function headerNameToString (value) { + return typeof value === 'string' + ? headerNameLowerCasedRecord[value] ?? value.toLowerCase() + : tree.lookup(value) ?? value.toString().toLowerCase() +} + +/** + * @param {Record | (Buffer | string | (Buffer | string)[])[]} headers + * @param {Record} [obj] + * @returns {Record} + */ +function parseHeaders (headers, obj) { // For H2 support if (!Array.isArray(headers)) return headers + if (obj === undefined) obj = {} for (let i = 0; i < headers.length; i += 2) { - const key = headers[i].toString() - const lowerCasedKey = headerNameLowerCasedRecord[key] ?? key.toLowerCase() - let val = obj[lowerCasedKey] + const key = headerNameToString(headers[i]) + let val = obj[key] if (!val) { const headersValue = headers[i + 1] if (typeof headersValue === 'string') { - obj[lowerCasedKey] = headersValue + obj[key] = headersValue } else { - obj[lowerCasedKey] = Array.isArray(headersValue) ? headersValue.map(x => x.toString('utf8')) : headersValue.toString('utf8') + obj[key] = Array.isArray(headersValue) ? headersValue.map(x => x.toString('utf8')) : headersValue.toString('utf8') } } else { - if (!Array.isArray(val)) { + if (typeof val === 'string') { val = [val] - obj[lowerCasedKey] = val + obj[key] = val } val.push(headers[i + 1].toString('utf8')) } @@ -461,6 +476,7 @@ module.exports = { isIterable, isAsyncIterable, isDestroyed, + headerNameToString, parseRawHeaders, parseHeaders, parseKeepAliveTimeout, diff --git a/package.json b/package.json index 0933d911830..05aa2050474 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "jest": "^29.0.2", "jsdom": "^23.0.0", "jsfuzz": "^1.0.15", + "mitata": "^0.1.6", "mocha": "^10.0.0", "mockttp": "^3.9.2", "p-timeout": "^3.2.0", diff --git a/test/tree.js b/test/tree.js new file mode 100644 index 00000000000..2a2342a1961 --- /dev/null +++ b/test/tree.js @@ -0,0 +1,40 @@ +'use strict' + +const { TernarySearchTree } = require('../lib/core/tree') +const { test } = require('tap') + +test('Ternary Search Tree', (t) => { + t.plan(1) + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' + const charactersLength = characters.length + + function generateAsciiString (length) { + let result = '' + for (let i = 0; i < length; ++i) { + result += characters[Math.floor(Math.random() * charactersLength)] + } + return result + } + const tst = new TernarySearchTree() + + const LENGTH = 5000 + + /** @type {string[]} */ + const random = new Array(LENGTH) + /** @type {Buffer[]} */ + const randomBuffer = new Array(LENGTH) + + for (let i = 0; i < LENGTH; ++i) { + const key = generateAsciiString((Math.random() * 100 + 5) | 0) + const lowerCasedKey = random[i] = key.toLowerCase() + randomBuffer[i] = Buffer.from(key) + tst.insert(Buffer.from(lowerCasedKey), lowerCasedKey) + } + + t.test('all', (t) => { + t.plan(LENGTH) + for (let i = 0; i < LENGTH; ++i) { + t.equal(tst.lookup(randomBuffer[i]), random[i]) + } + }) +}) diff --git a/test/util.js b/test/util.js index 75a4d8c1617..71a63f5c8af 100644 --- a/test/util.js +++ b/test/util.js @@ -1,7 +1,6 @@ 'use strict' -const t = require('tap') -const { test } = t +const { test } = require('tap') const { Stream } = require('stream') const { EventEmitter } = require('events') @@ -125,5 +124,5 @@ test('buildURL', (t) => { test('headerNameLowerCasedRecord', (t) => { t.plan(1) - t.ok(typeof headerNameLowerCasedRecord.hasOwnProperty === 'undefined') + t.ok(typeof headerNameLowerCasedRecord.hasOwnProperty !== 'function') }) From b918b7aec7ca27417cae33f02522ee99921e1d30 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:59:48 +0900 Subject: [PATCH 08/19] perf: twice faster method check (#2495) --- lib/core/request.js | 2 +- lib/core/util.js | 48 +++++++++++++++++++++++++++++++++++++++++++++ lib/fetch/util.js | 48 +-------------------------------------------- 3 files changed, 50 insertions(+), 48 deletions(-) diff --git a/lib/core/request.js b/lib/core/request.js index b0bd870e33a..0f89f318568 100644 --- a/lib/core/request.js +++ b/lib/core/request.js @@ -80,7 +80,7 @@ class Request { if (typeof method !== 'string') { throw new InvalidArgumentError('method must be a string') - } else if (tokenRegExp.exec(method) === null) { + } else if (!util.isValidHTTPToken(method)) { throw new InvalidArgumentError('invalid request method') } diff --git a/lib/core/util.js b/lib/core/util.js index 75d31888221..49d1c9938ed 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -442,6 +442,52 @@ function toUSVString (val) { return `${val}` } +/** + * @see https://tools.ietf.org/html/rfc7230#section-3.2.6 + * @param {number} c + */ +function isTokenCharCode (c) { + switch (c) { + case 0x22: + case 0x28: + case 0x29: + case 0x2c: + case 0x2f: + case 0x3a: + case 0x3b: + case 0x3c: + case 0x3d: + case 0x3e: + case 0x3f: + case 0x40: + case 0x5b: + case 0x5c: + case 0x5d: + case 0x7b: + case 0x7d: + // DQUOTE and "(),/:;<=>?@[\]{}" + return false + default: + // VCHAR %x21-7E + return c >= 0x21 && c <= 0x7e + } +} + +/** + * @param {string} characters + */ +function isValidHTTPToken (characters) { + if (characters.length === 0) { + return false + } + for (let i = 0; i < characters.length; ++i) { + if (!isTokenCharCode(characters.charCodeAt(i))) { + return false + } + } + return true +} + // Parsed accordingly to RFC 9110 // https://www.rfc-editor.org/rfc/rfc9110#field.content-range function parseRangeHeader (range) { @@ -490,6 +536,8 @@ module.exports = { isFormDataLike, buildURL, addAbortListener, + isValidHTTPToken, + isTokenCharCode, parseRangeHeader, nodeMajor, nodeMinor, diff --git a/lib/fetch/util.js b/lib/fetch/util.js index b729d70d1e8..9843f97444b 100644 --- a/lib/fetch/util.js +++ b/lib/fetch/util.js @@ -3,7 +3,7 @@ const { redirectStatusSet, referrerPolicySet: referrerPolicyTokens, badPortsSet } = require('./constants') const { getGlobalOrigin } = require('./global') const { performance } = require('perf_hooks') -const { isBlobLike, toUSVString, ReadableStreamFrom } = require('../core/util') +const { isBlobLike, toUSVString, ReadableStreamFrom, isValidHTTPToken } = require('../core/util') const assert = require('assert') const { isUint8Array } = require('util/types') @@ -103,52 +103,6 @@ function isValidReasonPhrase (statusText) { return true } -/** - * @see https://tools.ietf.org/html/rfc7230#section-3.2.6 - * @param {number} c - */ -function isTokenCharCode (c) { - switch (c) { - case 0x22: - case 0x28: - case 0x29: - case 0x2c: - case 0x2f: - case 0x3a: - case 0x3b: - case 0x3c: - case 0x3d: - case 0x3e: - case 0x3f: - case 0x40: - case 0x5b: - case 0x5c: - case 0x5d: - case 0x7b: - case 0x7d: - // DQUOTE and "(),/:;<=>?@[\]{}" - return false - default: - // VCHAR %x21-7E - return c >= 0x21 && c <= 0x7e - } -} - -/** - * @param {string} characters - */ -function isValidHTTPToken (characters) { - if (characters.length === 0) { - return false - } - for (let i = 0; i < characters.length; ++i) { - if (!isTokenCharCode(characters.charCodeAt(i))) { - return false - } - } - return true -} - /** * @see https://fetch.spec.whatwg.org/#header-name * @param {string} potentialValue From e535a6493fed2924609a25671bf5ace53ff3020d Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Fri, 8 Dec 2023 23:26:40 +0900 Subject: [PATCH 09/19] refactor: remove Error.captureStackTrace (#2509) --- lib/core/errors.js | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/core/errors.js b/lib/core/errors.js index e9adb2064da..0d0b7f60bc2 100644 --- a/lib/core/errors.js +++ b/lib/core/errors.js @@ -11,7 +11,6 @@ class UndiciError extends Error { class ConnectTimeoutError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, ConnectTimeoutError) this.name = 'ConnectTimeoutError' this.message = message || 'Connect Timeout Error' this.code = 'UND_ERR_CONNECT_TIMEOUT' @@ -21,7 +20,6 @@ class ConnectTimeoutError extends UndiciError { class HeadersTimeoutError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, HeadersTimeoutError) this.name = 'HeadersTimeoutError' this.message = message || 'Headers Timeout Error' this.code = 'UND_ERR_HEADERS_TIMEOUT' @@ -31,7 +29,6 @@ class HeadersTimeoutError extends UndiciError { class HeadersOverflowError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, HeadersOverflowError) this.name = 'HeadersOverflowError' this.message = message || 'Headers Overflow Error' this.code = 'UND_ERR_HEADERS_OVERFLOW' @@ -41,7 +38,6 @@ class HeadersOverflowError extends UndiciError { class BodyTimeoutError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, BodyTimeoutError) this.name = 'BodyTimeoutError' this.message = message || 'Body Timeout Error' this.code = 'UND_ERR_BODY_TIMEOUT' @@ -51,7 +47,6 @@ class BodyTimeoutError extends UndiciError { class ResponseStatusCodeError extends UndiciError { constructor (message, statusCode, headers, body) { super(message) - Error.captureStackTrace(this, ResponseStatusCodeError) this.name = 'ResponseStatusCodeError' this.message = message || 'Response Status Code Error' this.code = 'UND_ERR_RESPONSE_STATUS_CODE' @@ -65,7 +60,6 @@ class ResponseStatusCodeError extends UndiciError { class InvalidArgumentError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, InvalidArgumentError) this.name = 'InvalidArgumentError' this.message = message || 'Invalid Argument Error' this.code = 'UND_ERR_INVALID_ARG' @@ -75,7 +69,6 @@ class InvalidArgumentError extends UndiciError { class InvalidReturnValueError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, InvalidReturnValueError) this.name = 'InvalidReturnValueError' this.message = message || 'Invalid Return Value Error' this.code = 'UND_ERR_INVALID_RETURN_VALUE' @@ -85,7 +78,6 @@ class InvalidReturnValueError extends UndiciError { class AbortError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, AbortError) this.name = 'AbortError' this.message = message || 'The operation was aborted' } @@ -94,7 +86,6 @@ class AbortError extends UndiciError { class RequestAbortedError extends AbortError { constructor (message) { super(message) - Error.captureStackTrace(this, RequestAbortedError) this.name = 'AbortError' this.message = message || 'Request aborted' this.code = 'UND_ERR_ABORTED' @@ -104,7 +95,6 @@ class RequestAbortedError extends AbortError { class InformationalError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, InformationalError) this.name = 'InformationalError' this.message = message || 'Request information' this.code = 'UND_ERR_INFO' @@ -114,7 +104,6 @@ class InformationalError extends UndiciError { class RequestContentLengthMismatchError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, RequestContentLengthMismatchError) this.name = 'RequestContentLengthMismatchError' this.message = message || 'Request body length does not match content-length header' this.code = 'UND_ERR_REQ_CONTENT_LENGTH_MISMATCH' @@ -124,7 +113,6 @@ class RequestContentLengthMismatchError extends UndiciError { class ResponseContentLengthMismatchError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, ResponseContentLengthMismatchError) this.name = 'ResponseContentLengthMismatchError' this.message = message || 'Response body length does not match content-length header' this.code = 'UND_ERR_RES_CONTENT_LENGTH_MISMATCH' @@ -134,7 +122,6 @@ class ResponseContentLengthMismatchError extends UndiciError { class ClientDestroyedError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, ClientDestroyedError) this.name = 'ClientDestroyedError' this.message = message || 'The client is destroyed' this.code = 'UND_ERR_DESTROYED' @@ -144,7 +131,6 @@ class ClientDestroyedError extends UndiciError { class ClientClosedError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, ClientClosedError) this.name = 'ClientClosedError' this.message = message || 'The client is closed' this.code = 'UND_ERR_CLOSED' @@ -154,7 +140,6 @@ class ClientClosedError extends UndiciError { class SocketError extends UndiciError { constructor (message, socket) { super(message) - Error.captureStackTrace(this, SocketError) this.name = 'SocketError' this.message = message || 'Socket error' this.code = 'UND_ERR_SOCKET' @@ -165,7 +150,6 @@ class SocketError extends UndiciError { class NotSupportedError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, NotSupportedError) this.name = 'NotSupportedError' this.message = message || 'Not supported error' this.code = 'UND_ERR_NOT_SUPPORTED' @@ -175,7 +159,6 @@ class NotSupportedError extends UndiciError { class BalancedPoolMissingUpstreamError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, NotSupportedError) this.name = 'MissingUpstreamError' this.message = message || 'No upstream has been added to the BalancedPool' this.code = 'UND_ERR_BPL_MISSING_UPSTREAM' @@ -185,7 +168,6 @@ class BalancedPoolMissingUpstreamError extends UndiciError { class HTTPParserError extends Error { constructor (message, code, data) { super(message) - Error.captureStackTrace(this, HTTPParserError) this.name = 'HTTPParserError' this.code = code ? `HPE_${code}` : undefined this.data = data ? data.toString() : undefined @@ -195,7 +177,6 @@ class HTTPParserError extends Error { class ResponseExceededMaxSizeError extends UndiciError { constructor (message) { super(message) - Error.captureStackTrace(this, ResponseExceededMaxSizeError) this.name = 'ResponseExceededMaxSizeError' this.message = message || 'Response content exceeded max size' this.code = 'UND_ERR_RES_EXCEEDED_MAX_SIZE' @@ -205,7 +186,6 @@ class ResponseExceededMaxSizeError extends UndiciError { class RequestRetryError extends UndiciError { constructor (message, code, { headers, data }) { super(message) - Error.captureStackTrace(this, RequestRetryError) this.name = 'RequestRetryError' this.message = message || 'Request retry error' this.code = 'UND_ERR_REQ_RETRY' From 8422aa988243fb4c6c37b78519954d7337cc240b Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Sat, 9 Dec 2023 00:16:19 +0900 Subject: [PATCH 10/19] perf: Improve processHeader (#2513) * perf: Improve processHeader * rewrite * use if statement * fixup --- lib/core/request.js | 66 +++++++++++++-------------------------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/lib/core/request.js b/lib/core/request.js index 0f89f318568..caaf70d36bb 100644 --- a/lib/core/request.js +++ b/lib/core/request.js @@ -7,17 +7,11 @@ const { const assert = require('assert') const { kHTTP2BuildRequest, kHTTP2CopyHeaders, kHTTP1BuildRequest } = require('./symbols') const util = require('./util') +const { headerNameLowerCasedRecord } = require('./constants') -// tokenRegExp and headerCharRegex have been lifted from +// headerCharRegex have been lifted from // https://github.com/nodejs/node/blob/main/lib/_http_common.js -/** - * Verifies that the given val is a valid HTTP token - * per the rules defined in RFC 7230 - * See https://tools.ietf.org/html/rfc7230#section-3.2.6 - */ -const tokenRegExp = /^[\^_`a-zA-Z\-0-9!#$%&'*+.|~]+$/ - /** * Matches if val contains an invalid field-vchar * field-value = *( field-content / obs-fold ) @@ -416,65 +410,41 @@ function processHeader (request, key, val, skipAppend = false) { return } - if ( - request.host === null && - key.length === 4 && - key.toLowerCase() === 'host' - ) { + let headerName = headerNameLowerCasedRecord[key] + + if (headerName === undefined) { + headerName = key.toLowerCase() + if (headerNameLowerCasedRecord[headerName] === undefined && !util.isValidHTTPToken(headerName)) { + throw new InvalidArgumentError('invalid header key') + } + } + + if (request.host === null && headerName === 'host') { if (headerCharRegex.exec(val) !== null) { throw new InvalidArgumentError(`invalid ${key} header`) } // Consumed by Client request.host = val - } else if ( - request.contentLength === null && - key.length === 14 && - key.toLowerCase() === 'content-length' - ) { + } else if (request.contentLength === null && headerName === 'content-length') { request.contentLength = parseInt(val, 10) if (!Number.isFinite(request.contentLength)) { throw new InvalidArgumentError('invalid content-length header') } - } else if ( - request.contentType === null && - key.length === 12 && - key.toLowerCase() === 'content-type' - ) { + } else if (request.contentType === null && headerName === 'content-type') { request.contentType = val if (skipAppend) request.headers[key] = processHeaderValue(key, val, skipAppend) else request.headers += processHeaderValue(key, val) - } else if ( - key.length === 17 && - key.toLowerCase() === 'transfer-encoding' - ) { - throw new InvalidArgumentError('invalid transfer-encoding header') - } else if ( - key.length === 10 && - key.toLowerCase() === 'connection' - ) { + } else if (headerName === 'transfer-encoding' || headerName === 'keep-alive' || headerName === 'upgrade') { + throw new InvalidArgumentError(`invalid ${headerName} header`) + } else if (headerName === 'connection') { const value = typeof val === 'string' ? val.toLowerCase() : null if (value !== 'close' && value !== 'keep-alive') { throw new InvalidArgumentError('invalid connection header') } else if (value === 'close') { request.reset = true } - } else if ( - key.length === 10 && - key.toLowerCase() === 'keep-alive' - ) { - throw new InvalidArgumentError('invalid keep-alive header') - } else if ( - key.length === 7 && - key.toLowerCase() === 'upgrade' - ) { - throw new InvalidArgumentError('invalid upgrade header') - } else if ( - key.length === 6 && - key.toLowerCase() === 'expect' - ) { + } else if (headerName === 'expect') { throw new NotSupportedError('expect header not supported') - } else if (tokenRegExp.exec(key) === null) { - throw new InvalidArgumentError('invalid header key') } else { if (Array.isArray(val)) { for (let i = 0; i < val.length; i++) { From 5d63bb3aa5b93d0e8075428a2c4bb65d661f78db Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Sun, 10 Dec 2023 00:04:56 +0900 Subject: [PATCH 11/19] perf: reduce `String#toLowerCase` call (#2516) --- lib/client.js | 13 +++--- lib/core/util.js | 10 ++++- lib/fetch/body.js | 2 +- lib/fetch/headers.js | 99 ++++++++++++++++++++++++++++++++++++++----- lib/fetch/index.js | 86 ++++++++++++++++++------------------- lib/fetch/response.js | 6 +-- lib/fetch/util.js | 10 ++--- 7 files changed, 158 insertions(+), 68 deletions(-) diff --git a/lib/client.js b/lib/client.js index 2fec533faf1..b83863e296b 100644 --- a/lib/client.js +++ b/lib/client.js @@ -765,11 +765,14 @@ class Parser { } const key = this.headers[len - 2] - if (key.length === 10 && key.toString().toLowerCase() === 'keep-alive') { - this.keepAlive += buf.toString() - } else if (key.length === 10 && key.toString().toLowerCase() === 'connection') { - this.connection += buf.toString() - } else if (key.length === 14 && key.toString().toLowerCase() === 'content-length') { + if (key.length === 10) { + const headerName = util.bufferToLowerCasedHeaderName(key) + if (headerName === 'keep-alive') { + this.keepAlive += buf.toString() + } else if (headerName === 'connection') { + this.connection += buf.toString() + } + } else if (key.length === 14 && util.bufferToLowerCasedHeaderName(key) === 'content-length') { this.contentLength += buf.toString() } diff --git a/lib/core/util.js b/lib/core/util.js index 49d1c9938ed..6c2e1e55d21 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -226,7 +226,14 @@ function parseKeepAliveTimeout (val) { function headerNameToString (value) { return typeof value === 'string' ? headerNameLowerCasedRecord[value] ?? value.toLowerCase() - : tree.lookup(value) ?? value.toString().toLowerCase() + : tree.lookup(value) ?? value.toString('latin1').toLowerCase() +} + +/** + * @param {Buffer} value + */ +function bufferToLowerCasedHeaderName (value) { + return tree.lookup(value) ?? value.toString('latin1').toLowerCase() } /** @@ -523,6 +530,7 @@ module.exports = { isAsyncIterable, isDestroyed, headerNameToString, + bufferToLowerCasedHeaderName, parseRawHeaders, parseHeaders, parseKeepAliveTimeout, diff --git a/lib/fetch/body.js b/lib/fetch/body.js index 2a8b38f7415..6202887289a 100644 --- a/lib/fetch/body.js +++ b/lib/fetch/body.js @@ -374,7 +374,7 @@ function bodyMixinMethods (instance) { // If mimeType’s essence is "multipart/form-data", then: if (/multipart\/form-data/.test(contentType)) { const headers = {} - for (const [key, value] of this.headers) headers[key.toLowerCase()] = value + for (const [key, value] of this.headers) headers[key] = value const responseFormData = new FormData() diff --git a/lib/fetch/headers.js b/lib/fetch/headers.js index 2f1c0be5a47..883d7637cbf 100644 --- a/lib/fetch/headers.js +++ b/lib/fetch/headers.js @@ -135,13 +135,22 @@ class HeadersList { } } - // https://fetch.spec.whatwg.org/#header-list-contains + /** + * @see https://fetch.spec.whatwg.org/#header-list-contains + * @param {string} name + */ contains (name) { // A header list list contains a header name name if list // contains a header whose name is a byte-case-insensitive // match for name. - name = name.toLowerCase() + return this[kHeadersMap].has(name.toLowerCase()) + } + + /** + * @param {string} name + */ + lowerCaseContains (name) { return this[kHeadersMap].has(name) } @@ -151,7 +160,11 @@ class HeadersList { this.cookies = null } - // https://fetch.spec.whatwg.org/#concept-header-list-append + /** + * @see https://fetch.spec.whatwg.org/#concept-header-list-append + * @param {string} name + * @param {string} value + */ append (name, value) { this[kHeadersSortedMap] = null @@ -172,12 +185,42 @@ class HeadersList { } if (lowercaseName === 'set-cookie') { - this.cookies ??= [] - this.cookies.push(value) + (this.cookies ??= []).push(value) + } + } + + /** + * @param {string} name + * @param {string} value + */ + lowerCaseAppend (name, value) { + this[kHeadersSortedMap] = null + + // 1. If list contains name, then set name to the first such + // header’s name. + const exists = this[kHeadersMap].get(name) + + // 2. Append (name, value) to list. + if (exists) { + const delimiter = name === 'cookie' ? '; ' : ', ' + this[kHeadersMap].set(name, { + name: exists.name, + value: `${exists.value}${delimiter}${value}` + }) + } else { + this[kHeadersMap].set(name, { name, value }) + } + + if (name === 'set-cookie') { + (this.cookies ??= []).push(value) } } - // https://fetch.spec.whatwg.org/#concept-header-list-set + /** + * @see https://fetch.spec.whatwg.org/#concept-header-list-set + * @param {string} name + * @param {string} value + */ set (name, value) { this[kHeadersSortedMap] = null const lowercaseName = name.toLowerCase() @@ -193,11 +236,35 @@ class HeadersList { this[kHeadersMap].set(lowercaseName, { name, value }) } - // https://fetch.spec.whatwg.org/#concept-header-list-delete + /** + * @param {string} name + * @param {string} value + */ + lowerCaseSet (name, value) { + if (name === 'set-cookie') { + this.cookies = [value] + } + + // 1. If list contains name, then set the value of + // the first such header to value and remove the + // others. + // 2. Otherwise, append header (name, value) to list. + this[kHeadersMap].set(name, { name, value }) + } + + /** + * @see https://fetch.spec.whatwg.org/#concept-header-list-delete + * @param {string} name + */ delete (name) { - this[kHeadersSortedMap] = null + return this.lowerCaseDelete(name.toLowerCase()) + } - name = name.toLowerCase() + /** + * @param {string} name + */ + lowerCaseDelete (name) { + this[kHeadersSortedMap] = null if (name === 'set-cookie') { this.cookies = null @@ -206,7 +273,11 @@ class HeadersList { this[kHeadersMap].delete(name) } - // https://fetch.spec.whatwg.org/#concept-header-list-get + /** + * @see https://fetch.spec.whatwg.org/#concept-header-list-get + * @param {string} name + * @returns {string | null} + */ get (name) { const value = this[kHeadersMap].get(name.toLowerCase()) @@ -217,6 +288,14 @@ class HeadersList { return value === undefined ? null : value.value } + /** + * @param {string} name + * @returns {string | null} + */ + lowerCaseGet (name) { + return this[kHeadersMap].get(name)?.value ?? null + } + * [Symbol.iterator] () { // use the lowercased name for (const [name, { value }] of this[kHeadersMap]) { diff --git a/lib/fetch/index.js b/lib/fetch/index.js index b3b6e71eb79..0e5fc35a13d 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -57,7 +57,7 @@ const { const { kHeadersList, kConstruct } = require('../core/symbols') const EE = require('events') const { Readable, pipeline } = require('stream') -const { addAbortListener, isErrored, isReadable, nodeMajor, nodeMinor } = require('../core/util') +const { addAbortListener, isErrored, isReadable, nodeMajor, nodeMinor, bufferToLowerCasedHeaderName } = require('../core/util') const { dataURLProcessor, serializeAMimeType, parseMIMEType } = require('./dataURL') const { getGlobalDispatcher } = require('../global') const { webidl } = require('./webidl') @@ -475,7 +475,7 @@ function fetching ({ } // 12. If request’s header list does not contain `Accept`, then: - if (!request.headersList.contains('accept')) { + if (!request.headersList.lowerCaseContains('accept')) { // 1. Let value be `*/*`. const value = '*/*' @@ -492,14 +492,14 @@ function fetching ({ // TODO // 3. Append `Accept`/value to request’s header list. - request.headersList.append('accept', value) + request.headersList.lowerCaseAppend('accept', value) } // 13. If request’s header list does not contain `Accept-Language`, then // user agents should append `Accept-Language`/an appropriate value to // request’s header list. - if (!request.headersList.contains('accept-language')) { - request.headersList.append('accept-language', '*') + if (!request.headersList.lowerCaseContains('accept-language')) { + request.headersList.lowerCaseAppend('accept-language', '*') } // 14. If request’s priority is null, then use request’s initiator and @@ -718,7 +718,7 @@ async function mainFetch (fetchParams, recursive = false) { response.type === 'opaque' && internalResponse.status === 206 && internalResponse.rangeRequested && - !request.headers.contains('range') + !request.headers.lowerCaseContains('range') ) { response = internalResponse = makeNetworkError() } @@ -840,7 +840,7 @@ function schemeFetch (fetchParams) { // 8. If request’s header list does not contain `Range`: // 9. Otherwise: - if (!request.headersList.contains('range')) { + if (!request.headersList.lowerCaseContains('range')) { // 1. Let bodyWithType be the result of safely extracting blob. // Note: in the FileAPI a blob "object" is a Blob *or* a MediaSource. // In node, this can only ever be a Blob. Therefore we can safely @@ -854,14 +854,14 @@ function schemeFetch (fetchParams) { response.body = bodyWithType[0] // 4. Set response’s header list to « (`Content-Length`, serializedFullLength), (`Content-Type`, type) ». - response.headersList.set('content-length', serializedFullLength) - response.headersList.set('content-type', type) + response.headersList.lowerCaseSet('content-length', serializedFullLength) + response.headersList.lowerCaseSet('content-type', type) } else { // 1. Set response’s range-requested flag. response.rangeRequested = true // 2. Let rangeHeader be the result of getting `Range` from request’s header list. - const rangeHeader = request.headersList.get('range') + const rangeHeader = request.headersList.lowerCaseGet('range') // 3. Let rangeValue be the result of parsing a single range header value given rangeHeader and true. const rangeValue = simpleRangeHeaderValue(rangeHeader, true) @@ -921,9 +921,9 @@ function schemeFetch (fetchParams) { // 15. Set response’s header list to « (`Content-Length`, serializedSlicedLength), // (`Content-Type`, type), (`Content-Range`, contentRange) ». - response.headersList.set('content-length', serializedSlicedLength) - response.headersList.set('content-type', type) - response.headersList.set('content-range', contentRange) + response.headersList.lowerCaseSet('content-length', serializedSlicedLength) + response.headersList.lowerCaseSet('content-type', type) + response.headersList.lowerCaseSet('content-range', contentRange) } // 10. Return response. @@ -1040,7 +1040,7 @@ function fetchFinale (fetchParams, response) { responseStatus = response.status // 2. Let mimeType be the result of extracting a MIME type from response’s header list. - const mimeType = parseMIMEType(response.headersList.get('content-type')) // TODO: fix + const mimeType = parseMIMEType(response.headersList.lowerCaseGet('content-type')) // TODO: fix // 3. If mimeType is not failure, then set bodyInfo’s content type to the result of minimizing a supported MIME type given mimeType. if (mimeType !== 'failure') { @@ -1336,11 +1336,11 @@ function httpRedirectFetch (fetchParams, response) { // delete headerName from request’s header list. if (!sameOrigin(requestCurrentURL(request), locationURL)) { // https://fetch.spec.whatwg.org/#cors-non-wildcard-request-header-name - request.headersList.delete('authorization') + request.headersList.lowerCaseDelete('authorization') // "Cookie" and "Host" are forbidden request-headers, which undici doesn't implement. - request.headersList.delete('cookie') - request.headersList.delete('host') + request.headersList.lowerCaseDelete('cookie') + request.headersList.lowerCaseDelete('host') } // 14. If request’s body is non-null, then set request’s body to the first return @@ -1456,7 +1456,7 @@ async function httpNetworkOrCacheFetch ( // `Content-Length`/contentLengthHeaderValue to httpRequest’s header // list. if (contentLengthHeaderValue != null) { - httpRequest.headersList.append('content-length', contentLengthHeaderValue) + httpRequest.headersList.lowerCaseAppend('content-length', contentLengthHeaderValue) } // 9. If contentLengthHeaderValue is non-null, then append (`Content-Length`, @@ -1472,7 +1472,7 @@ async function httpNetworkOrCacheFetch ( // `Referer`/httpRequest’s referrer, serialized and isomorphic encoded, // to httpRequest’s header list. if (httpRequest.referrer instanceof URL) { - httpRequest.headersList.append('referer', isomorphicEncode(httpRequest.referrer.href)) + httpRequest.headersList.lowerCaseAppend('referer', isomorphicEncode(httpRequest.referrer.href)) } // 12. Append a request `Origin` header for httpRequest. @@ -1484,8 +1484,8 @@ async function httpNetworkOrCacheFetch ( // 14. If httpRequest’s header list does not contain `User-Agent`, then // user agents should append `User-Agent`/default `User-Agent` value to // httpRequest’s header list. - if (!httpRequest.headersList.contains('user-agent')) { - httpRequest.headersList.append('user-agent', typeof esbuildDetection === 'undefined' ? 'undici' : 'node') + if (!httpRequest.headersList.lowerCaseContains('user-agent')) { + httpRequest.headersList.lowerCaseAppend('user-agent', typeof esbuildDetection === 'undefined' ? 'undici' : 'node') } // 15. If httpRequest’s cache mode is "default" and httpRequest’s header @@ -1494,11 +1494,11 @@ async function httpNetworkOrCacheFetch ( // httpRequest’s cache mode to "no-store". if ( httpRequest.cache === 'default' && - (httpRequest.headersList.contains('if-modified-since') || - httpRequest.headersList.contains('if-none-match') || - httpRequest.headersList.contains('if-unmodified-since') || - httpRequest.headersList.contains('if-match') || - httpRequest.headersList.contains('if-range')) + (httpRequest.headersList.lowerCaseContains('if-modified-since') || + httpRequest.headersList.lowerCaseContains('if-none-match') || + httpRequest.headersList.lowerCaseContains('if-unmodified-since') || + httpRequest.headersList.lowerCaseContains('if-match') || + httpRequest.headersList.lowerCaseContains('if-range')) ) { httpRequest.cache = 'no-store' } @@ -1510,44 +1510,44 @@ async function httpNetworkOrCacheFetch ( if ( httpRequest.cache === 'no-cache' && !httpRequest.preventNoCacheCacheControlHeaderModification && - !httpRequest.headersList.contains('cache-control') + !httpRequest.headersList.lowerCaseContains('cache-control') ) { - httpRequest.headersList.append('cache-control', 'max-age=0') + httpRequest.headersList.lowerCaseAppend('cache-control', 'max-age=0') } // 17. If httpRequest’s cache mode is "no-store" or "reload", then: if (httpRequest.cache === 'no-store' || httpRequest.cache === 'reload') { // 1. If httpRequest’s header list does not contain `Pragma`, then append // `Pragma`/`no-cache` to httpRequest’s header list. - if (!httpRequest.headersList.contains('pragma')) { - httpRequest.headersList.append('pragma', 'no-cache') + if (!httpRequest.headersList.lowerCaseContains('pragma')) { + httpRequest.headersList.lowerCaseAppend('pragma', 'no-cache') } // 2. If httpRequest’s header list does not contain `Cache-Control`, // then append `Cache-Control`/`no-cache` to httpRequest’s header list. - if (!httpRequest.headersList.contains('cache-control')) { - httpRequest.headersList.append('cache-control', 'no-cache') + if (!httpRequest.headersList.lowerCaseContains('cache-control')) { + httpRequest.headersList.lowerCaseAppend('cache-control', 'no-cache') } } // 18. If httpRequest’s header list contains `Range`, then append // `Accept-Encoding`/`identity` to httpRequest’s header list. - if (httpRequest.headersList.contains('range')) { - httpRequest.headersList.append('accept-encoding', 'identity') + if (httpRequest.headersList.lowerCaseContains('range')) { + httpRequest.headersList.lowerCaseAppend('accept-encoding', 'identity') } // 19. Modify httpRequest’s header list per HTTP. Do not append a given // header if httpRequest’s header list contains that header’s name. // TODO: https://github.com/whatwg/fetch/issues/1285#issuecomment-896560129 - if (!httpRequest.headersList.contains('accept-encoding')) { + if (!httpRequest.headersList.lowerCaseContains('accept-encoding')) { if (urlHasHttpsScheme(requestCurrentURL(httpRequest))) { - httpRequest.headersList.append('accept-encoding', 'br, gzip, deflate') + httpRequest.headersList.lowerCaseAppend('accept-encoding', 'br, gzip, deflate') } else { - httpRequest.headersList.append('accept-encoding', 'gzip, deflate') + httpRequest.headersList.lowerCaseAppend('accept-encoding', 'gzip, deflate') } } - httpRequest.headersList.delete('host') + httpRequest.headersList.lowerCaseDelete('host') // 20. If includeCredentials is true, then: if (includeCredentials) { @@ -1630,7 +1630,7 @@ async function httpNetworkOrCacheFetch ( // 12. If httpRequest’s header list contains `Range`, then set response’s // range-requested flag. - if (httpRequest.headersList.contains('range')) { + if (httpRequest.headersList.lowerCaseContains('range')) { response.rangeRequested = true } @@ -2121,15 +2121,15 @@ async function httpNetworkFetch ( // We distinguish between them and iterate accordingly if (Array.isArray(rawHeaders)) { for (let i = 0; i < rawHeaders.length; i += 2) { - headersList.append(rawHeaders[i].toString('latin1'), rawHeaders[i + 1].toString('latin1')) + headersList.lowerCaseAppend(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1')) } - const contentEncoding = headersList.get('content-encoding') + const contentEncoding = headersList.lowerCaseGet('content-encoding') if (contentEncoding) { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 // "All content-coding values are case-insensitive..." codings = contentEncoding.toLowerCase().split(',').map((x) => x.trim()) } - location = headersList.get('location') + location = headersList.lowerCaseGet('location') } else { const keys = Object.keys(rawHeaders) for (let i = 0; i < keys.length; ++i) { @@ -2243,7 +2243,7 @@ async function httpNetworkFetch ( const headersList = new HeadersList() for (let i = 0; i < rawHeaders.length; i += 2) { - headersList.append(rawHeaders[i].toString('latin1'), rawHeaders[i + 1].toString('latin1')) + headersList.lowerCaseAppend(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1')) } resolve({ diff --git a/lib/fetch/response.js b/lib/fetch/response.js index b0b0a2ce543..54ef3595416 100644 --- a/lib/fetch/response.js +++ b/lib/fetch/response.js @@ -126,7 +126,7 @@ class Response { const value = isomorphicEncode(URLSerializer(parsedURL)) // 7. Append `Location`/value to responseObject’s response’s header list. - responseObject[kState].headersList.append('location', value) + responseObject[kState].headersList.lowerCaseAppend('location', value) // 8. Return responseObject. return responseObject @@ -496,8 +496,8 @@ function initializeResponse (response, init, body) { // 3. If body's type is non-null and response's header list does not contain // `Content-Type`, then append (`Content-Type`, body's type) to response's header list. - if (body.type != null && !response[kState].headersList.contains('Content-Type')) { - response[kState].headersList.append('content-type', body.type) + if (body.type != null && !response[kState].headersList.lowerCaseContains('content-type')) { + response[kState].headersList.lowerCaseAppend('content-type', body.type) } } } diff --git a/lib/fetch/util.js b/lib/fetch/util.js index 9843f97444b..13c4b468cc6 100644 --- a/lib/fetch/util.js +++ b/lib/fetch/util.js @@ -35,7 +35,7 @@ function responseLocationURL (response, requestFragment) { // 2. Let location be the result of extracting header list values given // `Location` and response’s header list. - let location = response.headersList.get('location') + let location = response.headersList.lowerCaseGet('location') // 3. If location is a header value, then set location to the result of // parsing location with response’s URL. @@ -153,7 +153,7 @@ function setRequestReferrerPolicyOnRedirect (request, actualResponse) { // 2. Let policy be the empty string. // 3. For each token in policy-tokens, if token is a referrer policy and token is not the empty string, then set policy to token. // 4. Return policy. - const policyHeader = (headersList.get('referrer-policy') ?? '').split(',') + const policyHeader = (headersList.lowerCaseGet('referrer-policy') ?? '').split(',') // Note: As the referrer-policy can contain multiple policies // separated by comma, we need to loop through all of them @@ -212,7 +212,7 @@ function appendFetchMetadata (httpRequest) { header = httpRequest.mode // 4. Set a structured field value `Sec-Fetch-Mode`/header in r’s header list. - httpRequest.headersList.set('sec-fetch-mode', header) + httpRequest.headersList.lowerCaseSet('sec-fetch-mode', header) // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-site-header // TODO @@ -229,7 +229,7 @@ function appendRequestOriginHeader (request) { // 2. If request’s response tainting is "cors" or request’s mode is "websocket", then append (`Origin`, serializedOrigin) to request’s header list. if (request.responseTainting === 'cors' || request.mode === 'websocket') { if (serializedOrigin) { - request.headersList.append('origin', serializedOrigin) + request.headersList.lowerCaseAppend('origin', serializedOrigin) } // 3. Otherwise, if request’s method is neither `GET` nor `HEAD`, then: @@ -260,7 +260,7 @@ function appendRequestOriginHeader (request) { if (serializedOrigin) { // 2. Append (`Origin`, serializedOrigin) to request’s header list. - request.headersList.append('origin', serializedOrigin) + request.headersList.lowerCaseAppend('origin', serializedOrigin) } } } From d434bb7b6468eb5179e8af0f42ac56969b7be3cd Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Sun, 10 Dec 2023 00:18:21 +0900 Subject: [PATCH 12/19] perf: optimize consumeEnd (#2510) --- lib/api/readable.js | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/api/readable.js b/lib/api/readable.js index e6e71c4554c..8ea6d512f60 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -6,9 +6,7 @@ const assert = require('assert') const { Readable } = require('stream') const { RequestAbortedError, NotSupportedError, InvalidArgumentError, AbortError } = require('../core/errors') const util = require('../core/util') -const { ReadableStreamFrom, toUSVString } = require('../core/util') - -let Blob +const { ReadableStreamFrom } = require('../core/util') const kConsume = Symbol('kConsume') const kReading = Symbol('kReading') @@ -267,14 +265,35 @@ function consumeStart (consume) { } } +/** + * @param {Buffer[]} chunks + * @param {number} length + */ +function chunksDecode (chunks, length) { + if (chunks.length === 0 || length === 0) { + return '' + } + const buffer = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks, length) + + const start = + buffer.length >= 3 && + // Skip BOM. + buffer[0] === 0xef && + buffer[1] === 0xbb && + buffer[2] === 0xbf + ? 3 + : 0 + return buffer.utf8Slice(start, buffer.length - start) +} + function consumeEnd (consume) { const { type, body, resolve, stream, length } = consume try { if (type === 'text') { - resolve(toUSVString(Buffer.concat(body))) + resolve(chunksDecode(body, length)) } else if (type === 'json') { - resolve(JSON.parse(Buffer.concat(body))) + resolve(JSON.parse(chunksDecode(body, length))) } else if (type === 'arrayBuffer') { const dst = new Uint8Array(length) @@ -286,9 +305,6 @@ function consumeEnd (consume) { resolve(dst.buffer) } else if (type === 'blob') { - if (!Blob) { - Blob = require('buffer').Blob - } resolve(new Blob(body, { type: stream[kContentType] })) } From 3fd7bf3a34f593c7d4fb03e48559755c659d1f1a Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:09:19 +0900 Subject: [PATCH 13/19] perf: reduce tst built time (#2517) --- lib/core/tree.js | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/lib/core/tree.js b/lib/core/tree.js index aa1641d217f..a61de9efbd9 100644 --- a/lib/core/tree.js +++ b/lib/core/tree.js @@ -1,6 +1,9 @@ 'use strict' -const { wellknownHeaderNames } = require('./constants') +const { + wellknownHeaderNames, + headerNameLowerCasedRecord +} = require('./constants') class TstNode { /** @type {any} */ @@ -16,14 +19,15 @@ class TstNode { /** * @param {Uint8Array} key * @param {any} value + * @param {number} index */ - constructor (key, value) { - if (key.length === 0) { + constructor (key, value, index) { + if (index === undefined || index >= key.length) { throw new TypeError('Unreachable') } - this.code = key[0] - if (key.length > 1) { - this.middle = new TstNode(key.subarray(1), value) + this.code = key[index] + if (key.length !== ++index) { + this.middle = new TstNode(key, value, index) } else { this.value = value } @@ -32,31 +36,32 @@ class TstNode { /** * @param {Uint8Array} key * @param {any} value + * @param {number} index */ - add (key, value) { - if (key.length === 0) { + add (key, value, index) { + if (index === undefined || index >= key.length) { throw new TypeError('Unreachable') } - const code = key[0] + const code = key[index] if (this.code === code) { - if (key.length === 1) { + if (key.length === ++index) { this.value = value } else if (this.middle !== null) { - this.middle.add(key.subarray(1), value) + this.middle.add(key, value, index) } else { - this.middle = new TstNode(key.subarray(1), value) + this.middle = new TstNode(key, value, index) } } else if (this.code < code) { if (this.left !== null) { - this.left.add(key, value) + this.left.add(key, value, index) } else { - this.left = new TstNode(key, value) + this.left = new TstNode(key, value, index) } } else { if (this.right !== null) { - this.right.add(key, value) + this.right.add(key, value, index) } else { - this.right = new TstNode(key, value) + this.right = new TstNode(key, value, index) } } } @@ -102,9 +107,9 @@ class TernarySearchTree { * */ insert (key, value) { if (this.node === null) { - this.node = new TstNode(key, value) + this.node = new TstNode(key, value, 0) } else { - this.node.add(key, value) + this.node.add(key, value, 0) } } @@ -119,7 +124,7 @@ class TernarySearchTree { const tree = new TernarySearchTree() for (let i = 0; i < wellknownHeaderNames.length; ++i) { - const key = wellknownHeaderNames[i].toLowerCase() + const key = headerNameLowerCasedRecord[wellknownHeaderNames[i]] tree.insert(Buffer.from(key), key) } From eaefcf807a445499dfd23aee0d8ec0babcd83799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Wed, 13 Dec 2023 23:07:44 +0100 Subject: [PATCH 14/19] feat: allow customization of build environment (#2403) This allows for the WASM artifacts to be built elsewhere than only in the alpine-based node container. --- build/wasm.js | 72 +++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/build/wasm.js b/build/wasm.js index fd90ac26fc9..2b63f3c7ab1 100644 --- a/build/wasm.js +++ b/build/wasm.js @@ -9,6 +9,18 @@ const WASM_SRC = resolve(__dirname, '../deps/llhttp') const WASM_OUT = resolve(__dirname, '../lib/llhttp') const DOCKERFILE = resolve(__dirname, './Dockerfile') +// These are defined by build environment +const WASM_CC = process.env.WASM_CC || 'clang' +let WASM_CFLAGS = process.env.WASM_CFLAGS || '--sysroot=/usr/share/wasi-sysroot -target wasm32-unknown-wasi' +let WASM_LDFLAGS = process.env.WASM_LDFLAGS || '' +const WASM_LDLIBS = process.env.WASM_LDLIBS || '' + +// These are relevant for undici and should not be overridden +WASM_CFLAGS += ' -Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor' +WASM_LDFLAGS += ' -Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all' +WASM_LDFLAGS += ' -Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table' +WASM_LDFLAGS += ' -Wl,--export=malloc -Wl,--export=free -Wl,--no-entry' + let platform = process.env.WASM_PLATFORM if (!platform && process.argv[2]) { platform = execSync('docker info -f "{{.OSType}}/{{.Architecture}}"').toString().trim() @@ -35,35 +47,25 @@ if (process.argv[2] === '--docker') { process.exit(0) } -// Gather information about the tools used for the build -const buildInfo = execSync('apk info -v').toString() -if (!buildInfo.includes('wasi-sdk')) { - console.log('Failed to generate build environment information') - process.exit(-1) +const hasApk = (function () { + try { execSync('command -v apk'); return true } catch (error) { return false } +})() +if (hasApk) { + // Gather information about the tools used for the build + const buildInfo = execSync('apk info -v').toString() + if (!buildInfo.includes('wasi-sdk')) { + console.log('Failed to generate build environment information') + process.exit(-1) + } + writeFileSync(join(WASM_OUT, 'wasm_build_env.txt'), buildInfo) } -writeFileSync(join(WASM_OUT, 'wasm_build_env.txt'), buildInfo) // Build wasm binary -execSync(`clang \ - --sysroot=/usr/share/wasi-sysroot \ - -target wasm32-unknown-wasi \ - -Ofast \ - -fno-exceptions \ - -fvisibility=hidden \ - -mexec-model=reactor \ - -Wl,-error-limit=0 \ - -Wl,-O3 \ - -Wl,--lto-O3 \ - -Wl,--strip-all \ - -Wl,--allow-undefined \ - -Wl,--export-dynamic \ - -Wl,--export-table \ - -Wl,--export=malloc \ - -Wl,--export=free \ - -Wl,--no-entry \ +execSync(`${WASM_CC} ${WASM_CFLAGS} ${WASM_LDFLAGS} \ ${join(WASM_SRC, 'src')}/*.c \ -I${join(WASM_SRC, 'include')} \ - -o ${join(WASM_OUT, 'llhttp.wasm')}`, { stdio: 'inherit' }) + -o ${join(WASM_OUT, 'llhttp.wasm')} \ + ${WASM_LDLIBS}`, { stdio: 'inherit' }) const base64Wasm = readFileSync(join(WASM_OUT, 'llhttp.wasm')).toString('base64') writeFileSync( @@ -72,27 +74,11 @@ writeFileSync( ) // Build wasm simd binary -execSync(`clang \ - --sysroot=/usr/share/wasi-sysroot \ - -target wasm32-unknown-wasi \ - -msimd128 \ - -Ofast \ - -fno-exceptions \ - -fvisibility=hidden \ - -mexec-model=reactor \ - -Wl,-error-limit=0 \ - -Wl,-O3 \ - -Wl,--lto-O3 \ - -Wl,--strip-all \ - -Wl,--allow-undefined \ - -Wl,--export-dynamic \ - -Wl,--export-table \ - -Wl,--export=malloc \ - -Wl,--export=free \ - -Wl,--no-entry \ +execSync(`${WASM_CC} ${WASM_CFLAGS} -msimd128 ${WASM_LDFLAGS} \ ${join(WASM_SRC, 'src')}/*.c \ -I${join(WASM_SRC, 'include')} \ - -o ${join(WASM_OUT, 'llhttp_simd.wasm')}`, { stdio: 'inherit' }) + -o ${join(WASM_OUT, 'llhttp_simd.wasm')} \ + ${WASM_LDLIBS}`, { stdio: 'inherit' }) const base64WasmSimd = readFileSync(join(WASM_OUT, 'llhttp_simd.wasm')).toString('base64') writeFileSync( From e1ab8b9954de4483da7b11581eda619e76447aad Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:32:13 +0900 Subject: [PATCH 15/19] fix: clear cache (#2519) --- lib/fetch/headers.js | 103 +++++++++--------------------------------- lib/fetch/index.js | 84 +++++++++++++++++----------------- lib/fetch/response.js | 6 +-- lib/fetch/util.js | 10 ++-- test/fetch/headers.js | 9 ++++ 5 files changed, 80 insertions(+), 132 deletions(-) diff --git a/lib/fetch/headers.js b/lib/fetch/headers.js index 883d7637cbf..a905baef499 100644 --- a/lib/fetch/headers.js +++ b/lib/fetch/headers.js @@ -114,7 +114,7 @@ function appendHeader (headers, name, value) { // forbidden response-header name, return. // 7. Append (name, value) to headers’s header list. - return headers[kHeadersList].append(name, value) + return headers[kHeadersList].append(name, value, false) // 8. If headers’s guard is "request-no-cors", then remove // privileged no-CORS request headers from headers @@ -138,20 +138,14 @@ class HeadersList { /** * @see https://fetch.spec.whatwg.org/#header-list-contains * @param {string} name + * @param {boolean} isLowerCase */ - contains (name) { + contains (name, isLowerCase) { // A header list list contains a header name name if list // contains a header whose name is a byte-case-insensitive // match for name. - return this[kHeadersMap].has(name.toLowerCase()) - } - - /** - * @param {string} name - */ - lowerCaseContains (name) { - return this[kHeadersMap].has(name) + return this[kHeadersMap].has(isLowerCase ? name : name.toLowerCase()) } clear () { @@ -164,13 +158,14 @@ class HeadersList { * @see https://fetch.spec.whatwg.org/#concept-header-list-append * @param {string} name * @param {string} value + * @param {boolean} isLowerCase */ - append (name, value) { + append (name, value, isLowerCase) { this[kHeadersSortedMap] = null // 1. If list contains name, then set name to the first such // header’s name. - const lowercaseName = name.toLowerCase() + const lowercaseName = isLowerCase ? name : name.toLowerCase() const exists = this[kHeadersMap].get(lowercaseName) // 2. Append (name, value) to list. @@ -189,41 +184,15 @@ class HeadersList { } } - /** - * @param {string} name - * @param {string} value - */ - lowerCaseAppend (name, value) { - this[kHeadersSortedMap] = null - - // 1. If list contains name, then set name to the first such - // header’s name. - const exists = this[kHeadersMap].get(name) - - // 2. Append (name, value) to list. - if (exists) { - const delimiter = name === 'cookie' ? '; ' : ', ' - this[kHeadersMap].set(name, { - name: exists.name, - value: `${exists.value}${delimiter}${value}` - }) - } else { - this[kHeadersMap].set(name, { name, value }) - } - - if (name === 'set-cookie') { - (this.cookies ??= []).push(value) - } - } - /** * @see https://fetch.spec.whatwg.org/#concept-header-list-set * @param {string} name * @param {string} value + * @param {boolean} isLowerCase */ - set (name, value) { + set (name, value, isLowerCase) { this[kHeadersSortedMap] = null - const lowercaseName = name.toLowerCase() + const lowercaseName = isLowerCase ? name : name.toLowerCase() if (lowercaseName === 'set-cookie') { this.cookies = [value] @@ -236,35 +205,14 @@ class HeadersList { this[kHeadersMap].set(lowercaseName, { name, value }) } - /** - * @param {string} name - * @param {string} value - */ - lowerCaseSet (name, value) { - if (name === 'set-cookie') { - this.cookies = [value] - } - - // 1. If list contains name, then set the value of - // the first such header to value and remove the - // others. - // 2. Otherwise, append header (name, value) to list. - this[kHeadersMap].set(name, { name, value }) - } - /** * @see https://fetch.spec.whatwg.org/#concept-header-list-delete * @param {string} name + * @param {boolean} isLowerCase */ - delete (name) { - return this.lowerCaseDelete(name.toLowerCase()) - } - - /** - * @param {string} name - */ - lowerCaseDelete (name) { + delete (name, isLowerCase) { this[kHeadersSortedMap] = null + if (!isLowerCase) name = name.toLowerCase() if (name === 'set-cookie') { this.cookies = null @@ -276,24 +224,15 @@ class HeadersList { /** * @see https://fetch.spec.whatwg.org/#concept-header-list-get * @param {string} name + * @param {boolean} isLowerCase * @returns {string | null} */ - get (name) { - const value = this[kHeadersMap].get(name.toLowerCase()) - + get (name, isLowerCase) { // 1. If list does not contain name, then return null. // 2. Return the values of all headers in list whose name // is a byte-case-insensitive match for name, // separated from each other by 0x2C 0x20, in order. - return value === undefined ? null : value.value - } - - /** - * @param {string} name - * @returns {string | null} - */ - lowerCaseGet (name) { - return this[kHeadersMap].get(name)?.value ?? null + return this[kHeadersMap].get(isLowerCase ? name : name.toLowerCase())?.value ?? null } * [Symbol.iterator] () { @@ -383,14 +322,14 @@ class Headers { // 6. If this’s header list does not contain name, then // return. - if (!this[kHeadersList].contains(name)) { + if (!this[kHeadersList].contains(name, false)) { return } // 7. Delete name from this’s header list. // 8. If this’s guard is "request-no-cors", then remove // privileged no-CORS request headers from this. - this[kHeadersList].delete(name) + this[kHeadersList].delete(name, false) } // https://fetch.spec.whatwg.org/#dom-headers-get @@ -412,7 +351,7 @@ class Headers { // 2. Return the result of getting name from this’s header // list. - return this[kHeadersList].get(name) + return this[kHeadersList].get(name, false) } // https://fetch.spec.whatwg.org/#dom-headers-has @@ -434,7 +373,7 @@ class Headers { // 2. Return true if this’s header list contains name; // otherwise false. - return this[kHeadersList].contains(name) + return this[kHeadersList].contains(name, false) } // https://fetch.spec.whatwg.org/#dom-headers-set @@ -483,7 +422,7 @@ class Headers { // 7. Set (name, value) in this’s header list. // 8. If this’s guard is "request-no-cors", then remove // privileged no-CORS request headers from this - this[kHeadersList].set(name, value) + this[kHeadersList].set(name, value, false) } // https://fetch.spec.whatwg.org/#dom-headers-getsetcookie diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 0e5fc35a13d..4a25542c310 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -475,7 +475,7 @@ function fetching ({ } // 12. If request’s header list does not contain `Accept`, then: - if (!request.headersList.lowerCaseContains('accept')) { + if (!request.headersList.contains('accept', true)) { // 1. Let value be `*/*`. const value = '*/*' @@ -492,14 +492,14 @@ function fetching ({ // TODO // 3. Append `Accept`/value to request’s header list. - request.headersList.lowerCaseAppend('accept', value) + request.headersList.append('accept', value, true) } // 13. If request’s header list does not contain `Accept-Language`, then // user agents should append `Accept-Language`/an appropriate value to // request’s header list. - if (!request.headersList.lowerCaseContains('accept-language')) { - request.headersList.lowerCaseAppend('accept-language', '*') + if (!request.headersList.contains('accept-language', true)) { + request.headersList.append('accept-language', '*', true) } // 14. If request’s priority is null, then use request’s initiator and @@ -718,7 +718,7 @@ async function mainFetch (fetchParams, recursive = false) { response.type === 'opaque' && internalResponse.status === 206 && internalResponse.rangeRequested && - !request.headers.lowerCaseContains('range') + !request.headers.contains('range', true) ) { response = internalResponse = makeNetworkError() } @@ -840,7 +840,7 @@ function schemeFetch (fetchParams) { // 8. If request’s header list does not contain `Range`: // 9. Otherwise: - if (!request.headersList.lowerCaseContains('range')) { + if (!request.headersList.contains('range', true)) { // 1. Let bodyWithType be the result of safely extracting blob. // Note: in the FileAPI a blob "object" is a Blob *or* a MediaSource. // In node, this can only ever be a Blob. Therefore we can safely @@ -854,14 +854,14 @@ function schemeFetch (fetchParams) { response.body = bodyWithType[0] // 4. Set response’s header list to « (`Content-Length`, serializedFullLength), (`Content-Type`, type) ». - response.headersList.lowerCaseSet('content-length', serializedFullLength) - response.headersList.lowerCaseSet('content-type', type) + response.headersList.set('content-length', serializedFullLength, true) + response.headersList.set('content-type', type, true) } else { // 1. Set response’s range-requested flag. response.rangeRequested = true // 2. Let rangeHeader be the result of getting `Range` from request’s header list. - const rangeHeader = request.headersList.lowerCaseGet('range') + const rangeHeader = request.headersList.get('range', true) // 3. Let rangeValue be the result of parsing a single range header value given rangeHeader and true. const rangeValue = simpleRangeHeaderValue(rangeHeader, true) @@ -921,9 +921,9 @@ function schemeFetch (fetchParams) { // 15. Set response’s header list to « (`Content-Length`, serializedSlicedLength), // (`Content-Type`, type), (`Content-Range`, contentRange) ». - response.headersList.lowerCaseSet('content-length', serializedSlicedLength) - response.headersList.lowerCaseSet('content-type', type) - response.headersList.lowerCaseSet('content-range', contentRange) + response.headersList.set('content-length', serializedSlicedLength, true) + response.headersList.set('content-type', type, true) + response.headersList.set('content-range', contentRange, true) } // 10. Return response. @@ -1040,7 +1040,7 @@ function fetchFinale (fetchParams, response) { responseStatus = response.status // 2. Let mimeType be the result of extracting a MIME type from response’s header list. - const mimeType = parseMIMEType(response.headersList.lowerCaseGet('content-type')) // TODO: fix + const mimeType = parseMIMEType(response.headersList.get('content-type', true)) // TODO: fix // 3. If mimeType is not failure, then set bodyInfo’s content type to the result of minimizing a supported MIME type given mimeType. if (mimeType !== 'failure') { @@ -1336,11 +1336,11 @@ function httpRedirectFetch (fetchParams, response) { // delete headerName from request’s header list. if (!sameOrigin(requestCurrentURL(request), locationURL)) { // https://fetch.spec.whatwg.org/#cors-non-wildcard-request-header-name - request.headersList.lowerCaseDelete('authorization') + request.headersList.delete('authorization', true) // "Cookie" and "Host" are forbidden request-headers, which undici doesn't implement. - request.headersList.lowerCaseDelete('cookie') - request.headersList.lowerCaseDelete('host') + request.headersList.delete('cookie', true) + request.headersList.delete('host', true) } // 14. If request’s body is non-null, then set request’s body to the first return @@ -1456,7 +1456,7 @@ async function httpNetworkOrCacheFetch ( // `Content-Length`/contentLengthHeaderValue to httpRequest’s header // list. if (contentLengthHeaderValue != null) { - httpRequest.headersList.lowerCaseAppend('content-length', contentLengthHeaderValue) + httpRequest.headersList.append('content-length', contentLengthHeaderValue, true) } // 9. If contentLengthHeaderValue is non-null, then append (`Content-Length`, @@ -1472,7 +1472,7 @@ async function httpNetworkOrCacheFetch ( // `Referer`/httpRequest’s referrer, serialized and isomorphic encoded, // to httpRequest’s header list. if (httpRequest.referrer instanceof URL) { - httpRequest.headersList.lowerCaseAppend('referer', isomorphicEncode(httpRequest.referrer.href)) + httpRequest.headersList.append('referer', isomorphicEncode(httpRequest.referrer.href), true) } // 12. Append a request `Origin` header for httpRequest. @@ -1484,8 +1484,8 @@ async function httpNetworkOrCacheFetch ( // 14. If httpRequest’s header list does not contain `User-Agent`, then // user agents should append `User-Agent`/default `User-Agent` value to // httpRequest’s header list. - if (!httpRequest.headersList.lowerCaseContains('user-agent')) { - httpRequest.headersList.lowerCaseAppend('user-agent', typeof esbuildDetection === 'undefined' ? 'undici' : 'node') + if (!httpRequest.headersList.contains('user-agent', true)) { + httpRequest.headersList.append('user-agent', typeof esbuildDetection === 'undefined' ? 'undici' : 'node', true) } // 15. If httpRequest’s cache mode is "default" and httpRequest’s header @@ -1494,11 +1494,11 @@ async function httpNetworkOrCacheFetch ( // httpRequest’s cache mode to "no-store". if ( httpRequest.cache === 'default' && - (httpRequest.headersList.lowerCaseContains('if-modified-since') || - httpRequest.headersList.lowerCaseContains('if-none-match') || - httpRequest.headersList.lowerCaseContains('if-unmodified-since') || - httpRequest.headersList.lowerCaseContains('if-match') || - httpRequest.headersList.lowerCaseContains('if-range')) + (httpRequest.headersList.contains('if-modified-since', true) || + httpRequest.headersList.contains('if-none-match', true) || + httpRequest.headersList.contains('if-unmodified-since', true) || + httpRequest.headersList.contains('if-match', true) || + httpRequest.headersList.contains('if-range', true)) ) { httpRequest.cache = 'no-store' } @@ -1510,44 +1510,44 @@ async function httpNetworkOrCacheFetch ( if ( httpRequest.cache === 'no-cache' && !httpRequest.preventNoCacheCacheControlHeaderModification && - !httpRequest.headersList.lowerCaseContains('cache-control') + !httpRequest.headersList.contains('cache-control', true) ) { - httpRequest.headersList.lowerCaseAppend('cache-control', 'max-age=0') + httpRequest.headersList.append('cache-control', 'max-age=0', true) } // 17. If httpRequest’s cache mode is "no-store" or "reload", then: if (httpRequest.cache === 'no-store' || httpRequest.cache === 'reload') { // 1. If httpRequest’s header list does not contain `Pragma`, then append // `Pragma`/`no-cache` to httpRequest’s header list. - if (!httpRequest.headersList.lowerCaseContains('pragma')) { - httpRequest.headersList.lowerCaseAppend('pragma', 'no-cache') + if (!httpRequest.headersList.contains('pragma', true)) { + httpRequest.headersList.append('pragma', 'no-cache', true) } // 2. If httpRequest’s header list does not contain `Cache-Control`, // then append `Cache-Control`/`no-cache` to httpRequest’s header list. - if (!httpRequest.headersList.lowerCaseContains('cache-control')) { - httpRequest.headersList.lowerCaseAppend('cache-control', 'no-cache') + if (!httpRequest.headersList.contains('cache-control', true)) { + httpRequest.headersList.append('cache-control', 'no-cache', true) } } // 18. If httpRequest’s header list contains `Range`, then append // `Accept-Encoding`/`identity` to httpRequest’s header list. - if (httpRequest.headersList.lowerCaseContains('range')) { - httpRequest.headersList.lowerCaseAppend('accept-encoding', 'identity') + if (httpRequest.headersList.contains('range', true)) { + httpRequest.headersList.append('accept-encoding', 'identity', true) } // 19. Modify httpRequest’s header list per HTTP. Do not append a given // header if httpRequest’s header list contains that header’s name. // TODO: https://github.com/whatwg/fetch/issues/1285#issuecomment-896560129 - if (!httpRequest.headersList.lowerCaseContains('accept-encoding')) { + if (!httpRequest.headersList.contains('accept-encoding', true)) { if (urlHasHttpsScheme(requestCurrentURL(httpRequest))) { - httpRequest.headersList.lowerCaseAppend('accept-encoding', 'br, gzip, deflate') + httpRequest.headersList.append('accept-encoding', 'br, gzip, deflate', true) } else { - httpRequest.headersList.lowerCaseAppend('accept-encoding', 'gzip, deflate') + httpRequest.headersList.append('accept-encoding', 'gzip, deflate', true) } } - httpRequest.headersList.lowerCaseDelete('host') + httpRequest.headersList.delete('host', true) // 20. If includeCredentials is true, then: if (includeCredentials) { @@ -1630,7 +1630,7 @@ async function httpNetworkOrCacheFetch ( // 12. If httpRequest’s header list contains `Range`, then set response’s // range-requested flag. - if (httpRequest.headersList.lowerCaseContains('range')) { + if (httpRequest.headersList.contains('range', true)) { response.rangeRequested = true } @@ -2121,15 +2121,15 @@ async function httpNetworkFetch ( // We distinguish between them and iterate accordingly if (Array.isArray(rawHeaders)) { for (let i = 0; i < rawHeaders.length; i += 2) { - headersList.lowerCaseAppend(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1')) + headersList.append(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1'), true) } - const contentEncoding = headersList.lowerCaseGet('content-encoding') + const contentEncoding = headersList.get('content-encoding', true) if (contentEncoding) { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 // "All content-coding values are case-insensitive..." codings = contentEncoding.toLowerCase().split(',').map((x) => x.trim()) } - location = headersList.lowerCaseGet('location') + location = headersList.get('location', true) } else { const keys = Object.keys(rawHeaders) for (let i = 0; i < keys.length; ++i) { @@ -2243,7 +2243,7 @@ async function httpNetworkFetch ( const headersList = new HeadersList() for (let i = 0; i < rawHeaders.length; i += 2) { - headersList.lowerCaseAppend(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1')) + headersList.append(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1'), true) } resolve({ diff --git a/lib/fetch/response.js b/lib/fetch/response.js index 54ef3595416..f8894b692eb 100644 --- a/lib/fetch/response.js +++ b/lib/fetch/response.js @@ -126,7 +126,7 @@ class Response { const value = isomorphicEncode(URLSerializer(parsedURL)) // 7. Append `Location`/value to responseObject’s response’s header list. - responseObject[kState].headersList.lowerCaseAppend('location', value) + responseObject[kState].headersList.append('location', value, true) // 8. Return responseObject. return responseObject @@ -496,8 +496,8 @@ function initializeResponse (response, init, body) { // 3. If body's type is non-null and response's header list does not contain // `Content-Type`, then append (`Content-Type`, body's type) to response's header list. - if (body.type != null && !response[kState].headersList.lowerCaseContains('content-type')) { - response[kState].headersList.lowerCaseAppend('content-type', body.type) + if (body.type != null && !response[kState].headersList.contains('content-type', true)) { + response[kState].headersList.append('content-type', body.type, true) } } } diff --git a/lib/fetch/util.js b/lib/fetch/util.js index 13c4b468cc6..2693d05bfcf 100644 --- a/lib/fetch/util.js +++ b/lib/fetch/util.js @@ -35,7 +35,7 @@ function responseLocationURL (response, requestFragment) { // 2. Let location be the result of extracting header list values given // `Location` and response’s header list. - let location = response.headersList.lowerCaseGet('location') + let location = response.headersList.get('location', true) // 3. If location is a header value, then set location to the result of // parsing location with response’s URL. @@ -153,7 +153,7 @@ function setRequestReferrerPolicyOnRedirect (request, actualResponse) { // 2. Let policy be the empty string. // 3. For each token in policy-tokens, if token is a referrer policy and token is not the empty string, then set policy to token. // 4. Return policy. - const policyHeader = (headersList.lowerCaseGet('referrer-policy') ?? '').split(',') + const policyHeader = (headersList.get('referrer-policy', true) ?? '').split(',') // Note: As the referrer-policy can contain multiple policies // separated by comma, we need to loop through all of them @@ -212,7 +212,7 @@ function appendFetchMetadata (httpRequest) { header = httpRequest.mode // 4. Set a structured field value `Sec-Fetch-Mode`/header in r’s header list. - httpRequest.headersList.lowerCaseSet('sec-fetch-mode', header) + httpRequest.headersList.set('sec-fetch-mode', header, true) // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-site-header // TODO @@ -229,7 +229,7 @@ function appendRequestOriginHeader (request) { // 2. If request’s response tainting is "cors" or request’s mode is "websocket", then append (`Origin`, serializedOrigin) to request’s header list. if (request.responseTainting === 'cors' || request.mode === 'websocket') { if (serializedOrigin) { - request.headersList.lowerCaseAppend('origin', serializedOrigin) + request.headersList.append('origin', serializedOrigin, true) } // 3. Otherwise, if request’s method is neither `GET` nor `HEAD`, then: @@ -260,7 +260,7 @@ function appendRequestOriginHeader (request) { if (serializedOrigin) { // 2. Append (`Origin`, serializedOrigin) to request’s header list. - request.headersList.lowerCaseAppend('origin', serializedOrigin) + request.headersList.append('origin', serializedOrigin, true) } } } diff --git a/test/fetch/headers.js b/test/fetch/headers.js index 48461103d78..fcf6f5f4891 100644 --- a/test/fetch/headers.js +++ b/test/fetch/headers.js @@ -741,3 +741,12 @@ tap.test('Headers.prototype.getSetCookie', (t) => { t.end() }) + +tap.test('When the value is updated, update the cache', (t) => { + t.plan(2) + const expected = [['a', 'a'], ['b', 'b'], ['c', 'c']] + const headers = new Headers(expected) + t.same([...headers], expected) + headers.append('d', 'd') + t.same([...headers], [...expected, ['d', 'd']]) +}) From 871baa74ec326518a3de3ee4591459e25617402e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torstein=20Bj=C3=B8rnstad?= Date: Tue, 19 Dec 2023 20:31:09 +0100 Subject: [PATCH 16/19] feat: Add resource timing entries for connection, request and response (#2481) --- docs/api/Dispatcher.md | 1 + lib/client.js | 2 + lib/core/request.js | 4 ++ lib/fetch/index.js | 19 +++++++ lib/fetch/util.js | 34 ++++++++++- test/client-dispatch.js | 103 ++++++++++++++++++++++++++++++++++ test/fetch/resource-timing.js | 66 ++++++++++++++++++++++ types/dispatcher.d.ts | 2 + 8 files changed, 229 insertions(+), 2 deletions(-) diff --git a/docs/api/Dispatcher.md b/docs/api/Dispatcher.md index fd463bfea16..0c678fc8623 100644 --- a/docs/api/Dispatcher.md +++ b/docs/api/Dispatcher.md @@ -209,6 +209,7 @@ Returns: `Boolean` - `false` if dispatcher is busy and further dispatch calls wo * **onConnect** `(abort: () => void, context: object) => void` - Invoked before request is dispatched on socket. May be invoked multiple times when a request is retried when the request at the head of the pipeline fails. * **onError** `(error: Error) => void` - Invoked when an error has occurred. May not throw. * **onUpgrade** `(statusCode: number, headers: Buffer[], socket: Duplex) => void` (optional) - Invoked when request is upgraded. Required if `DispatchOptions.upgrade` is defined or `DispatchOptions.method === 'CONNECT'`. +* **onResponseStarted** `() => void` (optional) - Invoked when response is received, before headers have been read. * **onHeaders** `(statusCode: number, headers: Buffer[], resume: () => void, statusText: string) => boolean` - Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. Not required for `upgrade` requests. * **onData** `(chunk: Buffer) => boolean` - Invoked when response payload data is received. Not required for `upgrade` requests. * **onComplete** `(trailers: Buffer[]) => void` - Invoked when response payload and trailers have been received and the request has completed. Not required for `upgrade` requests. diff --git a/lib/client.js b/lib/client.js index b83863e296b..bd4a400ad72 100644 --- a/lib/client.js +++ b/lib/client.js @@ -740,6 +740,7 @@ class Parser { if (!request) { return -1 } + request.onResponseStarted() } onHeaderField (buf) { @@ -1786,6 +1787,7 @@ function writeH2 (client, session, request) { stream.once('response', headers => { const { [HTTP2_HEADER_STATUS]: statusCode, ...realHeaders } = headers + request.onResponseStarted() if (request.onHeaders(Number(statusCode), realHeaders, stream.resume.bind(stream), '') === false) { stream.pause() diff --git a/lib/core/request.js b/lib/core/request.js index caaf70d36bb..fe63434ea98 100644 --- a/lib/core/request.js +++ b/lib/core/request.js @@ -253,6 +253,10 @@ class Request { } } + onResponseStarted () { + return this[kHandler].onResponseStarted?.() + } + onHeaders (statusCode, headers, resume, statusText) { assert(!this.aborted) assert(!this.completed) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 4a25542c310..d64dd90596c 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -41,6 +41,7 @@ const { urlIsLocal, urlIsHttpHttpsScheme, urlHasHttpsScheme, + clampAndCoursenConnectionTimingInfo, simpleRangeHeaderValue, buildContentRange } = require('./util') @@ -2098,12 +2099,30 @@ async function httpNetworkFetch ( // TODO (fix): Do we need connection here? const { connection } = fetchParams.controller + // Set timingInfo’s final connection timing info to the result of calling clamp and coarsen + // connection timing info with connection’s timing info, timingInfo’s post-redirect start + // time, and fetchParams’s cross-origin isolated capability. + // TODO: implement connection timing + timingInfo.finalConnectionTimingInfo = clampAndCoursenConnectionTimingInfo(undefined, timingInfo.postRedirectStartTime, fetchParams.crossOriginIsolatedCapability) + if (connection.destroyed) { abort(new DOMException('The operation was aborted.', 'AbortError')) } else { fetchParams.controller.on('terminated', abort) this.abort = connection.abort = abort } + + // Set timingInfo’s final network-request start time to the coarsened shared current time given + // fetchParams’s cross-origin isolated capability. + timingInfo.finalNetworkRequestStartTime = coarsenedSharedCurrentTime(fetchParams.crossOriginIsolatedCapability) + }, + + onResponseStarted () { + // Set timingInfo’s final network-response start time to the coarsened shared current + // time given fetchParams’s cross-origin isolated capability, immediately after the + // user agent’s HTTP parser receives the first byte of the response (e.g., frame header + // bytes for HTTP/2 or response status line for HTTP/1.x). + timingInfo.finalNetworkResponseStartTime = coarsenedSharedCurrentTime(fetchParams.crossOriginIsolatedCapability) }, onHeaders (status, rawHeaders, resume, statusText) { diff --git a/lib/fetch/util.js b/lib/fetch/util.js index 2693d05bfcf..32983720cc0 100644 --- a/lib/fetch/util.js +++ b/lib/fetch/util.js @@ -265,9 +265,38 @@ function appendRequestOriginHeader (request) { } } -function coarsenedSharedCurrentTime (crossOriginIsolatedCapability) { +// https://w3c.github.io/hr-time/#dfn-coarsen-time +function coarsenTime (timestamp, crossOriginIsolatedCapability) { // TODO - return performance.now() + return timestamp +} + +// https://fetch.spec.whatwg.org/#clamp-and-coarsen-connection-timing-info +function clampAndCoursenConnectionTimingInfo (connectionTimingInfo, defaultStartTime, crossOriginIsolatedCapability) { + if (!connectionTimingInfo?.startTime || connectionTimingInfo.startTime < defaultStartTime) { + return { + domainLookupStartTime: defaultStartTime, + domainLookupEndTime: defaultStartTime, + connectionStartTime: defaultStartTime, + connectionEndTime: defaultStartTime, + secureConnectionStartTime: defaultStartTime, + ALPNNegotiatedProtocol: connectionTimingInfo?.ALPNNegotiatedProtocol + } + } + + return { + domainLookupStartTime: coarsenTime(connectionTimingInfo.domainLookupStartTime, crossOriginIsolatedCapability), + domainLookupEndTime: coarsenTime(connectionTimingInfo.domainLookupEndTime, crossOriginIsolatedCapability), + connectionStartTime: coarsenTime(connectionTimingInfo.connectionStartTime, crossOriginIsolatedCapability), + connectionEndTime: coarsenTime(connectionTimingInfo.connectionEndTime, crossOriginIsolatedCapability), + secureConnectionStartTime: coarsenTime(connectionTimingInfo.secureConnectionStartTime, crossOriginIsolatedCapability), + ALPNNegotiatedProtocol: connectionTimingInfo.ALPNNegotiatedProtocol + } +} + +// https://w3c.github.io/hr-time/#dfn-coarsened-shared-current-time +function coarsenedSharedCurrentTime (crossOriginIsolatedCapability) { + return coarsenTime(performance.now(), crossOriginIsolatedCapability) } // https://fetch.spec.whatwg.org/#create-an-opaque-timing-info @@ -1145,6 +1174,7 @@ module.exports = { ReadableStreamFrom, toUSVString, tryUpgradeRequestToAPotentiallyTrustworthyURL, + clampAndCoursenConnectionTimingInfo, coarsenedSharedCurrentTime, determineRequestsReferrer, makePolicyContainer, diff --git a/test/client-dispatch.js b/test/client-dispatch.js index c3de37ae2a9..781118cc058 100644 --- a/test/client-dispatch.js +++ b/test/client-dispatch.js @@ -4,6 +4,8 @@ const { test } = require('tap') const http = require('http') const { Client, Pool, errors } = require('..') const stream = require('stream') +const { createSecureServer } = require('node:http2') +const pem = require('https-pem') test('dispatch invalid opts', (t) => { t.plan(14) @@ -813,3 +815,104 @@ test('dispatch onBodySent throws error', (t) => { }) }) }) + +test('dispatches in expected order', (t) => { + const server = http.createServer((req, res) => { + res.end('ended') + }) + t.teardown(server.close.bind(server)) + + server.listen(0, () => { + const client = new Pool(`http://localhost:${server.address().port}`) + + t.plan(1) + t.teardown(client.close.bind(client)) + + const dispatches = [] + + client.dispatch({ + path: '/', + method: 'POST', + body: 'body' + }, { + onConnect () { + dispatches.push('onConnect') + }, + onBodySent () { + dispatches.push('onBodySent') + }, + onResponseStarted () { + dispatches.push('onResponseStarted') + }, + onHeaders () { + dispatches.push('onHeaders') + }, + onData () { + dispatches.push('onData') + }, + onComplete () { + dispatches.push('onComplete') + t.same(dispatches, ['onConnect', 'onBodySent', 'onResponseStarted', 'onHeaders', 'onData', 'onComplete']) + }, + onError (err) { + t.error(err) + } + }) + }) +}) + +test('dispatches in expected order for http2', (t) => { + const server = createSecureServer(pem) + server.on('stream', (stream) => { + stream.respond({ + 'content-type': 'text/plain; charset=utf-8', + ':status': 200 + }) + stream.end('ended') + }) + + t.teardown(server.close.bind(server)) + + server.listen(0, () => { + const client = new Pool(`https://localhost:${server.address().port}`, { + connect: { + rejectUnauthorized: false + }, + allowH2: true + }) + + t.plan(1) + t.teardown(client.close.bind(client)) + + const dispatches = [] + + client.dispatch({ + path: '/', + method: 'POST', + body: 'body' + }, { + onConnect () { + dispatches.push('onConnect') + }, + onBodySent () { + dispatches.push('onBodySent') + }, + onResponseStarted () { + dispatches.push('onResponseStarted') + }, + onHeaders () { + dispatches.push('onHeaders') + }, + onData () { + dispatches.push('onData') + }, + onComplete () { + dispatches.push('onComplete') + t.same(dispatches, ['onConnect', 'onBodySent', 'onResponseStarted', 'onHeaders', 'onData', 'onComplete']) + }, + onError (err) { + t.error(err) + } + }) + }) +}) diff --git a/test/fetch/resource-timing.js b/test/fetch/resource-timing.js index 32753f57a86..b52e17e9073 100644 --- a/test/fetch/resource-timing.js +++ b/test/fetch/resource-timing.js @@ -70,3 +70,69 @@ test('should include encodedBodySize in performance entry', { skip }, (t) => { t.teardown(server.close.bind(server)) }) + +test('timing entries should be in order', { skip }, (t) => { + t.plan(13) + const obs = new PerformanceObserver(list => { + const [entry] = list.getEntries() + + t.ok(entry.startTime > 0) + t.ok(entry.fetchStart >= entry.startTime) + t.ok(entry.domainLookupStart >= entry.fetchStart) + t.ok(entry.domainLookupEnd >= entry.domainLookupStart) + t.ok(entry.connectStart >= entry.domainLookupEnd) + t.ok(entry.connectEnd >= entry.connectStart) + t.ok(entry.requestStart >= entry.connectEnd) + t.ok(entry.responseStart >= entry.requestStart) + t.ok(entry.responseEnd >= entry.responseStart) + t.ok(entry.duration > 0) + + t.ok(entry.redirectStart === 0) + t.ok(entry.redirectEnd === 0) + + obs.disconnect() + performance.clearResourceTimings() + }) + + obs.observe({ entryTypes: ['resource'] }) + + const server = createServer((req, res) => { + res.end('ok') + }).listen(0, async () => { + const body = await fetch(`http://localhost:${server.address().port}/redirect`) + t.strictSame('ok', await body.text()) + }) + + t.teardown(server.close.bind(server)) +}) + +test('redirect timing entries should be included when redirecting', { skip }, (t) => { + t.plan(4) + const obs = new PerformanceObserver(list => { + const [entry] = list.getEntries() + + t.ok(entry.redirectStart >= entry.startTime) + t.ok(entry.redirectEnd >= entry.redirectStart) + t.ok(entry.connectStart >= entry.redirectEnd) + + obs.disconnect() + performance.clearResourceTimings() + }) + + obs.observe({ entryTypes: ['resource'] }) + + const server = createServer((req, res) => { + if (req.url === '/redirect') { + res.statusCode = 307 + res.setHeader('location', '/redirect/') + res.end() + return + } + res.end('ok') + }).listen(0, async () => { + const body = await fetch(`http://localhost:${server.address().port}/redirect`) + t.strictSame('ok', await body.text()) + }) + + t.teardown(server.close.bind(server)) +}) diff --git a/types/dispatcher.d.ts b/types/dispatcher.d.ts index efc53eea791..24bf1519a25 100644 --- a/types/dispatcher.d.ts +++ b/types/dispatcher.d.ts @@ -210,6 +210,8 @@ declare namespace Dispatcher { onError?(err: Error): void; /** Invoked when request is upgraded either due to a `Upgrade` header or `CONNECT` method. */ onUpgrade?(statusCode: number, headers: Buffer[] | string[] | null, socket: Duplex): void; + /** Invoked when response is received, before headers have been read. **/ + onResponseStarted?(): void; /** Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. */ onHeaders?(statusCode: number, headers: Buffer[] | string[] | null, resume: () => void, statusText: string): boolean; /** Invoked when response payload data is received. */ From 974a85e9d423d0d3505adb39feef15434886f809 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Wed, 20 Dec 2023 14:58:55 +0100 Subject: [PATCH 17/19] =?UTF-8?q?Call=20fg.unregister()=20after=20a=20disp?= =?UTF-8?q?atcher=20is=20done,=20adds=20UNDICI=5FNO=5FFG=20to=E2=80=A6=20(?= =?UTF-8?q?#2527)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Call fg.unregister() after a dispatcher is done, adds UNDICI_NO_FG to disable finalization Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina --------- Signed-off-by: Matteo Collina --- lib/agent.js | 2 ++ lib/core/connect.js | 2 +- test/balanced-pool.js | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/agent.js b/lib/agent.js index 0b18f2a91bd..4ac34d26335 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -124,6 +124,7 @@ class Agent extends DispatcherBase { const client = ref.deref() /* istanbul ignore else: gc is undeterministic */ if (client) { + this[kFinalizer].unregister(client) closePromises.push(client.close()) } } @@ -137,6 +138,7 @@ class Agent extends DispatcherBase { const client = ref.deref() /* istanbul ignore else: gc is undeterministic */ if (client) { + this[kFinalizer].unregister(client) destroyPromises.push(client.destroy(err)) } } diff --git a/lib/core/connect.js b/lib/core/connect.js index 877956f7bee..2d2521dcfbd 100644 --- a/lib/core/connect.js +++ b/lib/core/connect.js @@ -15,7 +15,7 @@ let tls // include tls conditionally since it is not always available let SessionCache // FIXME: remove workaround when the Node bug is fixed // https://github.com/nodejs/node/issues/49344#issuecomment-1741776308 -if (global.FinalizationRegistry && !process.env.NODE_V8_COVERAGE) { +if (global.FinalizationRegistry && !(process.env.NODE_V8_COVERAGE || process.env.UNDICI_NO_FG)) { SessionCache = class WeakSessionCache { constructor (maxCachedSessions) { this._maxCachedSessions = maxCachedSessions diff --git a/test/balanced-pool.js b/test/balanced-pool.js index 7806652d95b..0e7c1bb169a 100644 --- a/test/balanced-pool.js +++ b/test/balanced-pool.js @@ -437,7 +437,7 @@ const cases = [ expectedRatios: [0.34, 0.34, 0.32], // Skip because the behavior of Node.js has changed - skip: nodeMajor >= 19 + skip: nodeMajor >= 18 }, // 8 From 38f2226718f8736bbb0d496f8b87618bcc78ddbc Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 20 Dec 2023 22:59:10 +0900 Subject: [PATCH 18/19] feat: expose headerNameToString (#2525) --- docs/api/Util.md | 25 +++++++++++++++++++++++++ index.js | 4 ++-- lib/core/util.js | 8 ++++++-- test/types/util.test-d.ts | 38 ++++++++++++++++++++++++++++++++++++++ types/index.d.ts | 1 + types/util.d.ts | 31 +++++++++++++++++++++++++++++++ 6 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 docs/api/Util.md create mode 100644 test/types/util.test-d.ts create mode 100644 types/util.d.ts diff --git a/docs/api/Util.md b/docs/api/Util.md new file mode 100644 index 00000000000..2393d079dfc --- /dev/null +++ b/docs/api/Util.md @@ -0,0 +1,25 @@ +# Util + +Utility API for third-party implementations of the dispatcher API. + +## `parseHeaders(headers, [obj])` + +Receives a header object and returns the parsed value. + +Arguments: + +- **headers** `Record | (Buffer | string | (Buffer | string)[])[]` (required) - Header object. + +- **obj** `Record` (optional) - Object to specify a proxy object. The parsed value is assigned to this object. But, if **headers** is an object, it is not used. + +Returns: `Record` If **headers** is an object, it is **headers**. Otherwise, if **obj** is specified, it is equivalent to **obj**. + +## `headerNameToString(value)` + +Retrieves a header name and returns its lowercase value. + +Arguments: + +- **value** `string | Buffer` (required) - Header name. + +Returns: `string` diff --git a/index.js b/index.js index 6898fb757d9..5fc0d9727ef 100644 --- a/index.js +++ b/index.js @@ -20,7 +20,6 @@ const { getGlobalDispatcher, setGlobalDispatcher } = require('./lib/global') const DecoratorHandler = require('./lib/handler/DecoratorHandler') const RedirectHandler = require('./lib/handler/RedirectHandler') const createRedirectInterceptor = require('./lib/interceptor/redirectInterceptor') -const { parseHeaders } = require('./lib/core/util') let hasCrypto try { @@ -47,7 +46,8 @@ module.exports.createRedirectInterceptor = createRedirectInterceptor module.exports.buildConnector = buildConnector module.exports.errors = errors module.exports.util = { - parseHeaders + parseHeaders: util.parseHeaders, + headerNameToString: util.headerNameToString } function makeDispatcher (fn) { diff --git a/lib/core/util.js b/lib/core/util.js index 6c2e1e55d21..7cd411d9f3e 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -221,7 +221,9 @@ function parseKeepAliveTimeout (val) { } /** - * @param {string | Buffer} value + * Retrieves a header name and returns its lowercase value. + * @param {string | Buffer} value Header name + * @returns {string} */ function headerNameToString (value) { return typeof value === 'string' @@ -230,7 +232,9 @@ function headerNameToString (value) { } /** - * @param {Buffer} value + * Receive the buffer as a string and return its lowercase value. + * @param {Buffer} value Header name + * @returns {string} */ function bufferToLowerCasedHeaderName (value) { return tree.lookup(value) ?? value.toString('latin1').toLowerCase() diff --git a/test/types/util.test-d.ts b/test/types/util.test-d.ts new file mode 100644 index 00000000000..9879fd31507 --- /dev/null +++ b/test/types/util.test-d.ts @@ -0,0 +1,38 @@ +import { expectAssignable } from 'tsd'; +import { util } from '../../types/util'; + +expectAssignable>( + util.parseHeaders({ 'content-type': 'text/plain' }) +); + +expectAssignable>( + //@ts-ignore + util.parseHeaders({ 'content-type': 'text/plain' }, {}) +); + +expectAssignable>( + util.parseHeaders({} as Record | string[], {}) +); + +expectAssignable>( + util.parseHeaders(['content-type', 'text/plain']) +); + +expectAssignable>( + util.parseHeaders([Buffer.from('content-type'), Buffer.from('text/plain')]) +); + +expectAssignable>( + util.parseHeaders( + [Buffer.from('content-type'), Buffer.from('text/plain')], + {} + ) +); + +expectAssignable>( + util.parseHeaders([Buffer.from('content-type'), [Buffer.from('text/plain')]]) +); + +expectAssignable(util.headerNameToString('content-type')); + +expectAssignable(util.headerNameToString(Buffer.from('content-type'))); diff --git a/types/index.d.ts b/types/index.d.ts index 0ea8bdc217d..8b35475219b 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -17,6 +17,7 @@ import ProxyAgent from'./proxy-agent' import RetryHandler from'./retry-handler' import { request, pipeline, stream, connect, upgrade } from './api' +export * from './util' export * from './cookies' export * from './fetch' export * from './file' diff --git a/types/util.d.ts b/types/util.d.ts new file mode 100644 index 00000000000..2a604148fd8 --- /dev/null +++ b/types/util.d.ts @@ -0,0 +1,31 @@ +export namespace util { + /** + * Retrieves a header name and returns its lowercase value. + * @param value Header name + */ + export function headerNameToString(value: string | Buffer): string; + + /** + * Receives a header object and returns the parsed value. + * @param headers Header object + */ + export function parseHeaders( + headers: + | Record + | (Buffer | string | (Buffer | string)[])[] + ): Record; + /** + * Receives a header object and returns the parsed value. + * @param headers Header object + * @param obj Object to specify a proxy object. Used to assign parsed values. But, if `headers` is an object, it is not used. + * @returns If `headers` is an object, it is `headers`. Otherwise, if `obj` is specified, it is equivalent to `obj`. + */ + export function parseHeaders< + H extends + | Record + | (Buffer | string | (Buffer | string)[])[] + >( + headers: H, + obj?: H extends any[] ? Record : never + ): Record; +} From 250b89af0ae27b93aaacbb885e852636e2c78ce6 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Wed, 20 Dec 2023 15:00:32 +0100 Subject: [PATCH 19/19] Bumped v6.1.0 Signed-off-by: Matteo Collina --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 05aa2050474..f4f135ce8fb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "undici", - "version": "6.0.1", + "version": "6.1.0", "description": "An HTTP/1.1 client, written from scratch for Node.js", "homepage": "https://undici.nodejs.org", "bugs": {