Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions laya-ts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,23 @@ out.usage.truncated_questions; // ids of the questions whose head left too lit
The fields are absent only where no state was ever encoded (empty question schema, or a
start hook that supplied the result).

### Collapsed options

The same budget also cuts the options themselves: each is capped at 48 tokens, and once they
overflow `head_max_len` all of them are re-capped at `max(4, (head_max_len - 16) // n)`. Two
options that share a prefix can come out of that cut as the *same* token span, so the question
can no longer name them apart while still answering normally — and an answer chosen from 42
distinguishable spans of 58 has an accuracy ceiling of 72% that nothing else in the response
mentions (issue #538; mirrors Python `laya.common.collapsed_options`):

```ts
out.usage.options; // absent when every option kept a span of its own
out.usage.options?.intent; // { total: 58, distinct: 42, tokens_per_option: 4 }
```

`total` is what the question defines, not the markers that reached the sequence, so a report
cannot read "43 of 43" about a question whose missing options never entered the input at all.

## Batching (many states, one call)

Port of `Agent.predict_batch` / `Router.route_batch` / `Router.predict_batch`. The throughput
Expand Down
11 changes: 10 additions & 1 deletion laya-ts/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
buildQuestionPrefix,
clampTemperature,
collateItems,
collapsedOptions,
confidenceFromProbs,
answerConfidence,
checkMinConfidence,
Expand All @@ -15,7 +16,7 @@ import {
softmax,
tempBucket,
} from "./common.js";
import type { SequenceStats } from "./common.js";
import type { OptionStats, SequenceStats } from "./common.js";
import type { Batch, SessionProvider } from "./providers.js";
import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";
import { decide, type DecideOptions, type DecisionResult } from "./structured.js";
Expand Down Expand Up @@ -86,6 +87,9 @@ export interface SystemUsage {
state_tokens_dropped?: number;
truncated?: boolean;
truncated_questions?: string[];
/** Present only when some question's options no longer have a token span each (issue #538,
* Python `collapsed_options`), keyed by question id. Absent when every option kept its own. */
options?: Record<string, OptionStats>;
}

export interface SystemOneResult {
Expand Down Expand Up @@ -675,6 +679,10 @@ export class Agent extends HookRegistry {
// it from the length of the state it sent (issue #174, Python #181 parity).
const st = built[s].stats;
const dropped = st.reduce((a, x) => Math.max(a, x.state_tokens_dropped), 0);
// Only when a question actually lost options to the head budget, as Python's
// `Agent.predict_batch` does: an answer chosen from 42 distinguishable spans of 58 has a
// ceiling the caller cannot otherwise see.
const collapsed = collapsedOptions(ids, st);
out.push({
model: "laya-rl-agent",
answers,
Expand All @@ -685,6 +693,7 @@ export class Agent extends HookRegistry {
state_tokens_dropped: dropped,
truncated: dropped > 0,
truncated_questions: ids.filter((_, qi) => st[qi].truncated),
...(Object.keys(collapsed).length ? { options: collapsed } : {}),
},
});
}
Expand Down
42 changes: 41 additions & 1 deletion laya-ts/src/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,21 @@ export function renderOptions(q: InternalQ): string[] {
labels.true + ": " + (t !== null && t !== undefined && t !== "" ? renderCriterion(t) : "yes, the statement holds"),
];
}
export interface OptionStats {
/** Options the question defines, not the markers that survived the sequence clamp. */
total: number;
/** How many of them still have a token span of their own. */
distinct: number;
/** The per-option cap the head budget forced, or null when none was applied. */
tokens_per_option: number | null;
}
export interface QuestionPrefix {
/** [CLS] head [SEP] options [SEP] — the state-independent part of the sequence. */
ids: number[];
/** Mask-marker positions (absolute; the prefix sits at the start of the final sequence). */
markers: number[];
nOptions: number;
optionStats: OptionStats;
}
/** The question half of `buildSequence`: everything before the state tokens. Hoisted out so
* callers asking several questions about the same state can encode the state text only once. */
Expand Down Expand Up @@ -103,17 +112,29 @@ function buildQuestionPrefixUncached(tok: TokenizerLike, q: InternalQ,
let optIds = order.map((i) =>
[tok.maskId, ...tok.encode(" " + opts[i].split(maskTok).join(" ")).slice(0, 48)]);
let budget = headMaxLen - optIds.reduce((a, o) => a + o.length, 0);
let tokensPerOption: number | null = null;
if (budget < 16) {
const per = Math.max(4, Math.floor((headMaxLen - 16) / Math.max(1, optIds.length)));
optIds = optIds.map((o) => o.slice(0, per));
budget = headMaxLen - optIds.reduce((a, o) => a + o.length, 0);
tokensPerOption = per;
}
headIds = headIds.slice(0, Math.max(8, budget));
const ids = [tok.clsId, ...headIds, tok.sepId];
const markers: number[] = [];
for (const o of optIds) { markers.push(ids.length); ids.push(...o); }
ids.push(tok.sepId);
return { ids, markers, nOptions: opts.length };
// Counted on the capped option ids, before assembly, exactly as Python's `build_head` does:
// re-slicing the finished sequence cannot close the last option's span, so the last option
// always looks distinguishable however it collided (#538).
return {
ids, markers, nOptions: opts.length,
optionStats: {
total: opts.length,
distinct: new Set(optIds.map((o) => o.join(","))).size,
tokens_per_option: tokensPerOption,
},
};
}
export interface SequenceStats {
/** Encoded length of the full state, before the window clamp. */
Expand All @@ -124,6 +145,8 @@ export interface SequenceStats {
state_tokens_dropped: number;
/** True when the clamp dropped any state token. */
truncated: boolean;
/** What the head budget did to this question's options (Python `build_sequence`'s other stats). */
options: OptionStats;
}

/** Append pre-encoded state tokens to a question prefix. Identical output to building the
Expand All @@ -149,9 +172,26 @@ export function sequenceWithState(prefix: QuestionPrefix, stateIds: number[], se
state_tokens_used: used,
state_tokens_dropped: stateIds.length - used,
truncated: used < stateIds.length,
options: prefix.optionStats,
},
};
}
/** The questions whose options no longer have a token span each, keyed by question id.
*
* `total` is what the question defines, not the number of markers that reached the sequence, so a
* report cannot say "43 of 43" about a question whose 28 missing options never entered the input
* at all. Mirrors Python `laya.common.collapsed_options`; empty when nothing collapsed, which is
* the overwhelming majority of requests, so the agents add the key only when it says something. */
export function collapsedOptions(qids: string[],
stats: (SequenceStats | undefined)[]): Record<string, OptionStats> {
const out: Record<string, OptionStats> = {};
qids.forEach((qid, i) => {
const s = stats[i]?.options;
if (s && s.distinct < s.total) out[qid] = { total: s.total, distinct: s.distinct,
tokens_per_option: s.tokens_per_option };
});
return out;
}
export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,
maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[]; stats: SequenceStats } {
const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));
Expand Down
3 changes: 2 additions & 1 deletion laya-ts/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ export {
buildSequence,
buildQuestionPrefix,
sequenceWithState,
collapsedOptions,
softmax,
confidenceFromProbs,
answerConfidence,
Expand All @@ -66,7 +67,7 @@ export {
TEMP_MIN,
TEMP_MAX,
} from "./common.js";
export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix, SequenceStats } from "./common.js";
export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix, SequenceStats, OptionStats } from "./common.js";
export { q, best, topK, isConfident } from "./dx.js";
export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";
export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js";
153 changes: 153 additions & 0 deletions laya-ts/tests/option-collapse.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
import { describe, expect, it } from "vitest";
import { Agent, defaultTokenizer } from "../src/agent.js";
import { buildSequence, collapsedOptions } from "../src/common.js";
import type { TokenizerLike } from "../src/tokenizer.js";

// TS mirror of tests/test_option_collapse.py (issue #538): options capped to fit the head budget
// can collapse into one token span, and the answer then comes from a question that can no longer
// name them apart. Python reports it as `build_sequence(..., return_stats=True)`'s third value and
// publishes it as `usage["options"]`; the TS port built the identical cap (same 48-token floor,
// same `max(4, (headMaxLen - 16) // n)` re-cap) but kept no count of what it cost.
//
// Every expected number below is what laya/common.py returns for the same stub tokenizer on
// `d8a2e59`, so a divergence here is a port bug rather than a style choice.

/** Character-level ids, so options that share a prefix share their leading tokens. */
const TOK: TokenizerLike = {
...defaultTokenizer(),
clsId: 2,
sepId: 3,
maskId: 1,
padId: 0,
maskToken: "[MASK]",
encode: (text: string) => [...text].map((c) => 10 + (c.charCodeAt(0) % 90)),
};

const qChoice = (labels: string[]) => ({
t: "choice",
ins: "which one?",
crit: Object.fromEntries(labels.map((k) => [k, k.replace(/_/g, " ")])),
}) as never;

// Four labels that agree for their first fourteen characters, as MASSIVE's iot_hue_light* do.
const HUE = ["iot_hue_lightup", "iot_hue_lightoff", "iot_hue_lightdim", "iot_hue_lightchange"];
const MANY = Array.from({ length: 40 }, (_, i) => `label_${String(i).padStart(2, "0")}`);

const built = (labels: string[], maxLen: number, headMaxLen: number, state = "a state") =>
buildSequence(TOK, state, qChoice(labels), maxLen, headMaxLen);
const statsFor = (labels: string[], maxLen: number, headMaxLen: number, state = "a state") =>
built(labels, maxLen, headMaxLen, state).stats;

describe("the option report's shape", () => {
it("carries exactly the three fields Python's stats dict carries", () => {
expect(Object.keys(statsFor(["yes", "no"], 128, 64).options).sort())
.toEqual(["distinct", "tokens_per_option", "total"]);
});

it("counts the options the question defines, not the markers that survived", () => {
// Python: 40 labels at maxLen 40 leave 2 markers in the sequence, and `total` stays 40 --
// a report counted from markers would read "2 of 2 distinct" about a question with 40 labels.
const wide = built(MANY, 40, 400);
expect(wide.stats.options.total).toBe(40);
expect(wide.markers.length).toBe(2);
expect(wide.stats.options).toEqual({ total: 40, distinct: 4, tokens_per_option: 9 });
});
});

describe("nothing collapses when the head has room", () => {
it("one span per option, and no cap applied", () => {
// Python: {"options": 4, "options_distinct": 4, "tokens_per_option": null} at (512, 256),
// assembled to 186 tokens.
expect(statsFor(HUE, 512, 256).options)
.toEqual({ total: 4, distinct: 4, tokens_per_option: null });
expect(built(HUE, 512, 256).ids.length).toBe(186);
});
});

describe("the cut collapses shared prefixes", () => {
it("applies the cap and loses three of the four spans", () => {
// Python: floor((24 - 16) / 4) = 2, raised to the floor of 4, so every option is cut to
// " iot" -- one span for four labels.
const tight = built(HUE, 128, 24);
expect(tight.stats.options).toEqual({ total: 4, distinct: 1, tokens_per_option: 4 });
expect(tight.markers.length).toBe(4);
expect(tight.ids.length).toBe(35);
});

it("is the question's number, not the state's", () => {
const other = statsFor(HUE, 128, 24, "a completely different request");
expect(other.options.distinct).toBe(statsFor(HUE, 128, 24).options.distinct);
});

it("keeps spans that differ, at the same budget", () => {
// The report is about the option texts: four labels that part after two characters all keep
// their own span under the same cap.
expect(statsFor(["alpha", "bravo", "charlie", "delta"], 128, 24).options)
.toEqual({ total: 4, distinct: 4, tokens_per_option: 4 });
});
});

describe("collapsedOptions filters as laya.common.collapsed_options does", () => {
const tight = statsFor(HUE, 128, 24);
const roomy = statsFor(HUE, 512, 256);

it("names only the questions that lost a span", () => {
expect(Object.keys(collapsedOptions(["intent", "dept"], [tight, roomy]))).toEqual(["intent"]);
expect(collapsedOptions(["intent"], [tight])["intent"])
.toEqual({ total: 4, distinct: 1, tokens_per_option: 4 });
});

it("is empty when nothing collapsed", () => {
expect(collapsedOptions(["dept"], [roomy])).toEqual({});
});

it("skips a state with no stats at all", () => {
expect(collapsedOptions(["a", "b"], [undefined, roomy])).toEqual({});
});
});

/** A real Agent whose model is a fake provider: only the reporting is under test. */
function makeAgent() {
const provider = {
async runEncoder(b: { inputIds: number[][] }) {
return { lastHidden: b.inputIds.map(() => [[1, 0], [0, 1]]) };
},
async runHead(_h: unknown, b: { inputIds: number[][], markerPos: number[][] }) {
return {
logits: b.inputIds.map((_r, i) => b.markerPos[i].map((_m, k) => 4 - k)),
act: b.inputIds.map(() => [3, 0]),
};
},
};
return new Agent({ provider, tok: TOK } as never);
}

const QUESTIONS = { intent: { type: "choice", instructions: "which one?",
criteria: Object.fromEntries(HUE.map((k) => [k, k.replace(/_/g, " ")])) } };
// Long enough that the 128-token window cannot keep it, so the clamp and the cap both bite.
const LONG_STATE = Array.from({ length: 200 }, () => "word").join(" ");

describe("what the Agent publishes", () => {
it("puts the collapse in usage.options when the head budget forced it", async () => {
const out = await makeAgent().systemOne("a state", QUESTIONS as never,
{ maxLen: 128, headMaxLen: 24 });
expect(out.usage.options).toEqual({ intent: { total: 4, distinct: 1, tokens_per_option: 4 } });
});

it("says nothing when every option kept its own span", async () => {
const out = await makeAgent().systemOne("a state", QUESTIONS as never,
{ maxLen: 512, headMaxLen: 256 });
expect(out.usage.options).toBeUndefined();
expect(Object.keys(out.usage!).sort()).toEqual(
["input_tokens", "output_tokens", "state_tokens", "state_tokens_dropped", "truncated",
"truncated_questions"]);
});

it("reports a cut state and a cut head as the two separate facts they are", async () => {
const out = await makeAgent().systemOne(LONG_STATE, QUESTIONS as never,
{ maxLen: 128, headMaxLen: 24 });
expect(out.usage.truncated).toBe(true);
expect(out.usage.truncated_questions).toEqual(["intent"]);
expect(out.usage.options).toEqual({ intent: { total: 4, distinct: 1, tokens_per_option: 4 } });
});
});
Loading