Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/guides/web/web-network-captures.md
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,20 @@ name decoding does not normalize the recorded paths used for grouping.
The inventory describes observed names, without inferring required parameters,
value types or a complete API schema. Sensitive-parent refusals still apply.

Select `include_request_header_names: true` in the view to include the header,
cookie and set-cookie names recorded for each group. The default and explicit
`false` omit the `request_header_names` field. Selected groups carry
`request_headers`, `cookie_names` and `set_cookie_names`, each an array of
`{name, record_ordinals}` entries sorted by Unicode code point and listed once
per record. `request_headers` holds the `request.headers` names verbatim, so
`Accept` and `accept` stay distinct, while credential header names such as
`Authorization` and `Cookie` appear as names only. `cookie_names` holds the
`request.cookies` names and `set_cookie_names` the `response.cookies` names.
No header, cookie or set-cookie value, prefix, digest or substring is copied,
including from headers whose value was redacted at capture time. A name's
presence does not establish authentication, session semantics, a required
header or a complete header schema.

| Rule | Exact segment format | Placeholder |
| ---------- | ---------------------------------------------------- | ------------ |
| `numeric` | ASCII digits only | `{numeric}` |
Expand Down
99 changes: 87 additions & 12 deletions src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@ import {
RECORD_SCRATCH_BYTES,
SEGMENT_BYTES,
readQueryParameterNames,
readRecordedHeaderNames,
readRecordedRequestLine,
scanRouteTemplate,
textBytes,
type RouteRule,
type RouteTemplate,
type RecordedHeaderNames,
type RecordedRequestLine,
} from "./webNetworkRouteTemplates.js";

Expand All @@ -47,6 +49,14 @@ interface GroupState {
readonly statuses: Set<number>;
readonly contentTypes: Set<string>;
readonly queryNames?: Map<string, number[]>;
readonly headers?: HeaderNames;
}
/** One name surface's retained name-to-ordinal map. */
type Names = Map<string, number[]>;
interface HeaderNames {
readonly request: Names;
readonly cookie: Names;
readonly setCookie: Names;
}
type Buckets = Map<string, Map<string, Map<string, Map<string, GroupState>>>>;

Expand All @@ -64,19 +74,29 @@ export interface ApiSurfaceComparisonGroup {
readonly exactPathOrdinal: (path: string) => number | undefined;
}

const sortedNames = (names: Names) =>
[...names]
.sort(([a], [b]) => compare(a, b))
.map(([name, record_ordinals]) => ({
name,
record_ordinals: [...record_ordinals],
}));

/** Own group identity, deduplication and admission for the entire projection. */
export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumulator {
readonly #buckets: Buckets = new Map();
readonly #groups: GroupState[] = [];
readonly #excluded: Exclusion[] = [];
#unavailableResponse = false;
#unreadableHeaderMember = false;

constructor(
maximumBytes = WEB_NETWORK_CAPTURE_LIMITS.outputBytes,
readonly includeQueryParameterNames = false,
private readonly comparison?: {
readonly operation: string;
},
readonly includeRequestHeaderNames = false,
) {
super(maximumBytes, "API-surface", comparison?.operation);
}
Expand Down Expand Up @@ -111,7 +131,17 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
if (route === null) return this.overBudget();
const queryNames = readQueryParameterNames(line.query, this.scratch);
if (queryNames === null) return this.overBudget();
appended = this.#append(record.ordinal, line, route, queryNames);
const headers = this.includeRequestHeaderNames
? readRecordedHeaderNames(record.reported, this.scratch)
: undefined;
if (headers === null) return this.overBudget();
appended = this.#append(
record.ordinal,
line,
route,
queryNames,
headers,
);
}
if (appended.ok) {
limitations.value();
Expand All @@ -129,6 +159,7 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
line: Extract<RecordedRequestLine, { readonly kind: "line" }>,
route: RouteTemplate,
queryNames: ReadonlySet<string> | undefined,
headers: RecordedHeaderNames | undefined,
): Result<void, AnalysisError> {
const { method, origin, path, status } = line;
const mime = mediaTypeEssence(line.rawMime);
Expand Down Expand Up @@ -166,6 +197,20 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
bytes += ENTRY_BYTES + textBytes(name.length);
}
}
if (headers !== undefined) {
if (group === undefined) bytes += CAPTURE_CONTAINER_BYTES * 3;
for (const [surface, names] of [
["request", headers.request],
["cookie", headers.cookie],
["setCookie", headers.setCookie],
] as const) {
for (const name of names) {
bytes += ENTRY_BYTES; // This name's source ordinal, once per record.
if (!group?.headers?.[surface].has(name))
bytes += ENTRY_BYTES + textBytes(name.length);
}
}
}
// Reserve the whole mutation while route scratch is still live. No sets,
// map entries, arrays or new observed values are retained before admission.
if (!this.retain(bytes)) return this.overBudget();
Expand All @@ -187,6 +232,15 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
statuses: new Set(),
contentTypes: new Set(),
...(queryNames === undefined ? {} : { queryNames: new Map() }),
...(headers === undefined
? {}
: {
headers: {
request: new Map(),
cookie: new Map(),
setCookie: new Map(),
},
}),
};
const byOrigin = this.#buckets.get(method) ?? new Map();
this.#buckets.set(method, byOrigin);
Expand All @@ -204,6 +258,7 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
if (group.paths !== undefined && !group.paths.has(path))
group.paths.set(path, ordinal);
if (status === null || mime === null) this.#unavailableResponse = true;
if (headers?.unreadableMember === true) this.#unreadableHeaderMember = true;
if (status !== null) group.statuses.add(status);
if (mime !== null) group.contentTypes.add(mime);
if (queryNames !== undefined && group.queryNames !== undefined) {
Expand All @@ -213,6 +268,17 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
else ordinals.push(ordinal);
}
}
if (group.headers !== undefined && headers !== undefined)
for (const [names, selected] of [
[group.headers.request, headers.request],
[group.headers.cookie, headers.cookie],
[group.headers.setCookie, headers.setCookie],
] as const)
for (const name of selected) {
const ordinals = names.get(name);
if (ordinals === undefined) names.set(name, [ordinal]);
else ordinals.push(ordinal);
}
return ok(undefined);
}

Expand Down Expand Up @@ -268,14 +334,16 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
statuses: [...group.statuses].sort((a, b) => a - b),
content_types: [...group.contentTypes].sort(compare),
...(group.queryNames === undefined
? {}
: { query_parameter_names: sortedNames(group.queryNames) }),
...(group.headers === undefined
? {}
: {
query_parameter_names: [...group.queryNames]
.sort(([a], [b]) => compare(a, b))
.map(([name, record_ordinals]) => ({
name,
record_ordinals: [...record_ordinals],
})),
request_header_names: {
request_headers: sortedNames(group.headers.request),
cookie_names: sortedNames(group.headers.cookie),
set_cookie_names: sortedNames(group.headers.setCookie),
},
}),
flags: [
...(group.ordinals.length === 1 ? ["single-sample" as const] : []),
Expand All @@ -294,11 +362,18 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
),
excluded_records: [...this.#excluded],
limitations: this.sortedLimitations(),
unknowns: this.#unavailableResponse
? [
"Some retained records have unavailable status or content type values.",
]
: [],
unknowns: [
...(this.#unavailableResponse
? [
"Some retained records have unavailable status or content type values.",
]
: []),
...(this.#unreadableHeaderMember
? [
"Some retained header, cookie or set-cookie members have no string name, so the selected name lists are incomplete.",
]
: []),
],
};
}
}
27 changes: 19 additions & 8 deletions src/domain/analysisView/apiSurfaceSchema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,25 @@ export const apiSurfaceViewSchema = z.strictObject({
.describe(
"Include distinct observed query parameter names and their original record ordinals; names do not establish required parameters or a complete API schema.",
),
include_request_header_names: z
.boolean()
.optional()
.describe(
"Include the request, cookie and set-cookie header names recorded for each group, with the records that presented them. No header, cookie or set-cookie value is copied, and no name establishes authentication, session semantics or a complete header schema.",
),
});
const ruleSchema = z.enum(ROUTE_RULES);
const nameOrdinalsSchema = z.array(
z.strictObject({
name: z.string(),
record_ordinals: z.array(z.int().nonnegative()),
}),
);
const requestHeaderNamesSchema = z.strictObject({
request_headers: nameOrdinalsSchema,
cookie_names: nameOrdinalsSchema,
set_cookie_names: nameOrdinalsSchema,
});
const groupSchema = z.strictObject({
method: z.string(),
origin: z.string(),
Expand All @@ -28,14 +45,8 @@ const groupSchema = z.strictObject({
record_ordinals: z.array(z.int().nonnegative()),
statuses: z.array(z.number()),
content_types: z.array(z.string()),
query_parameter_names: z
.array(
z.strictObject({
name: z.string(),
record_ordinals: z.array(z.int().nonnegative()),
}),
)
.optional(),
query_parameter_names: nameOrdinalsSchema.optional(),
request_header_names: requestHeaderNamesSchema.optional(),
flags: z.array(
z.enum(["single-sample", "mixed-statuses", "mixed-content-types"]),
),
Expand Down
Loading
Loading