From a0d27f370abc83769e1b91773efb9354c5536d59 Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:42:30 +0545 Subject: [PATCH] fix(ts): normalise the action head at the model boundary, not in the decoder laya/agent.py::_forward returns torch.softmax(act, -1) and _decode_answers reads act[r, 0] as a probability; laya/onnx_agent.py does the same in numpy and the Java decoder's parameter is actionProbabilities. laya-ts softmaxed inside _decodeRow, so the decode layer could not satisfy laya-java/fixtures/decode.json at all: 0 of its 40 cases agreed on action.act_probability. It now reads the row the way the reference does, and tests/decode-parity.test.ts asserts 36 of the 40 answers field-for-field (the four option_order cases stay open: the port has no permutation step in either layer). --- laya-ts/src/agent.ts | 19 +++- laya-ts/tests/decode-parity.test.ts | 152 ++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 laya-ts/tests/decode-parity.test.ts diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts index dfa8077ea..31a2f8421 100644 --- a/laya-ts/src/agent.ts +++ b/laya-ts/src/agent.ts @@ -686,7 +686,13 @@ export class Agent extends HookRegistry { for (let qi = 0; qi < ids.length; qi++) { answers[ids[qi]] = this._decodeRow( logits[row] as number[], - (act[row] as number[]) ?? [1, 0], + // The action head is normalised here, at the model boundary, as `laya/agent.py`'s + // `_forward` (agent.py:1329) and `laya/onnx_agent.py` (onnx_agent.py:805) do: the decode + // layer's contract -- the Java decoder's `actionProbabilities`, and the `act` rows in + // `laya-java/fixtures/decode.json` -- is a probability row. Softmaxing it a second time + // inside `_decodeRow` reported 0.6899 for the row `[0.9, 0.1]`, where the reference + // reports 0.9. + softmax((act[row] as number[]) ?? [1, 0]), internals[qi], built[s].items[qi].markers.length, opts.lang ?? null, @@ -720,7 +726,11 @@ export class Agent extends HookRegistry { return out; } - /** Decode one head row into the answer for its question. */ + /** + * Decode one head row into the answer for its question. `actRow` is the model boundary's + * normalised action-probability row, so its first entry is the reported `act_probability` -- + * exactly what `Agent._decode_answers` reads as `act[r, 0]`. + */ private _decodeRow( logitsRow: number[], actRow: number[], @@ -739,8 +749,9 @@ export class Agent extends HookRegistry { : (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0); const z = logitsRow.slice(0, k).map((v) => v / scale); const p = softmax(z); - const actP = softmax(actRow.slice(0, Math.max(2, actRow.length))); - const ext = { act_probability: r4(actP[0]) }; + // Read, not re-normalised: the row arrives as probabilities. The `slice(0, Math.max(2, ...))` + // this replaces could only pad an Act row that validation had already refused for being empty. + const ext = { act_probability: r4(actRow[0]) }; // Same quantity on every question type (max(p)), so callers can gate across types on // one number; `confidence` stays as-is for existing callers (entropy for choice/score). // Histogram-binning recalibration (issue #871, Python parity) recalibrates ans_raw diff --git a/laya-ts/tests/decode-parity.test.ts b/laya-ts/tests/decode-parity.test.ts new file mode 100644 index 000000000..deb0ca43c --- /dev/null +++ b/laya-ts/tests/decode-parity.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { Agent } from "../src/agent.js"; + +// `laya-java/fixtures/decode.json` is the shared decode oracle: `gen_fixtures.py:1845-1860` hands +// each synthesised row to laya's own `Agent._decode_answers` and records the answer, so the same +// file is what the Java port asserts against field-by-field at tolerance 0.0 +// (`laya-java/.../decode/DecoderParityTest.java`). The rows hit branches rather than look +// realistic -- every temperature bucket, a top-2 tie, an exactly uniform 20-option softmax, +/-700 +// magnitudes that overflow a naive `exp`, a logits block three columns wider than the option +// count, and a per-language override -- which is the point: none of it needs a checkpoint. +// +// `option_order` is out of scope here. The port has no un-permutation step in the decode layer and +// no permutation step in the prompt layer, so the two permuted cases per checkpoint +// (`choice-4-permuted`, `score-permuted`) cannot agree on `choice`/`probabilities`/`score`. Their +// permutation-invariant fields are still asserted below, so the rest of the row stays covered. +const fixture = JSON.parse( + readFileSync(new URL("../../laya-java/fixtures/decode.json", import.meta.url), "utf8"), +) as Record; + +type DecodeCase = { + question: { t: "choice" | "score" | "noul"; ins: string; crit?: unknown; labels?: unknown }; + option_order: number[] | null; + lang: string | null; + logits: number[]; + logits_block_width: number; + act: number[]; + answer: Record; +}; + +type DecodeCheckpoint = { + config: { + temperature: number[]; + temperature_by_options: Record; + lang_temperatures_raw: Record< + string, + { temperature: number[]; temperature_by_options: Record } + >; + }; + qtypes: Record; + cases: Record; +}; + +const fakeProvider = () => ({ + async runEncoder(_b: unknown) { return { lastHidden: [[1, 0], [0, 1]] }; }, + async runHead(_h: unknown) { return { logits: [[2, 0]], act: [[3, 0]] }; }, +}); + +/** The Agent the reference's `shim` is: the checkpoint's clamped scales and one language override. */ +function agentFor(ck: DecodeCheckpoint): Agent { + return new Agent({ + provider: fakeProvider(), + cfg: { + temperature: ck.config.temperature, + temperature_by_options: ck.config.temperature_by_options, + }, + lang_temperatures: ck.config.lang_temperatures_raw, + } as never); +} + +/** The row as the head returns it: padded past the option count with large finite values. */ +function paddedRow(c: DecodeCase): number[] { + const row = [...c.logits]; + while (row.length < c.logits_block_width) row.push(99.0); + return row; +} + +function decodeRow(agent: Agent, c: DecodeCase): Record { + const decodeRow = (agent as unknown as { + _decodeRow( + logits: number[], + act: number[], + q: DecodeCase["question"], + k: number, + lang: string | null, + ): Record; + })._decodeRow; + return decodeRow.call(agent, paddedRow(c), c.act, c.question, c.logits.length, c.lang); +} + +describe("decode.json parity", () => { + for (const [checkpointName, ck] of Object.entries(fixture)) { + if ("skipped" in (ck as unknown as Record)) continue; + const agent = agentFor(ck); + describe(checkpointName, () => { + for (const [caseName, c] of Object.entries(ck.cases)) { + if (c.option_order !== null) continue; + it(`${caseName} decodes to the reference answer`, () => { + expect(decodeRow(agent, c)).toEqual(c.answer); + }); + } + }); + } +}); + +describe("decode.json parity across option_order", () => { + for (const [checkpointName, ck] of Object.entries(fixture)) { + if ("skipped" in (ck as unknown as Record)) continue; + const agent = agentFor(ck); + for (const [caseName, c] of Object.entries(ck.cases)) { + if (c.option_order === null) continue; + it(`${checkpointName}/${caseName} keeps the permutation-invariant fields`, () => { + const got = decodeRow(agent, c); + expect(got.type).toBe(c.answer.type); + expect(got.confidence).toBe(c.answer.confidence); + expect(got.answer_confidence).toBe(c.answer.answer_confidence); + expect(got.action).toEqual(c.answer.action); + }); + } + } +}); + +describe("the decode layer reads the action row as a probability", () => { + // `laya/agent.py:1329` (`_forward` returns `torch.softmax(act, -1)`) and `laya/onnx_agent.py:805` + // normalise the action head at the model boundary; `Agent._decode_answers` then reports + // `round(float(act[r, 0]), 4)` (`agent.py:1365`), and the Java decoder takes the pair as + // `actionProbabilities` (`DecoderParityTest.java:71-72`). Normalising inside the decode layer + // instead re-normalised the row: softmax([0.9, 0.1])[0] = 0.6899. + const ck = fixture.english; + const agent = agentFor(ck); + const caseOf = (act: number[]): DecodeCase => ({ + ...ck.cases["choice-2"], + act, + }); + + it("reports the first entry of the row the reference reports", () => { + expect(decodeRow(agent, caseOf([0.9, 0.1])).action).toEqual({ act_probability: 0.9 }); + expect(decodeRow(agent, caseOf([0.99, 0.01])).action).toEqual({ act_probability: 0.99 }); + expect(decodeRow(agent, caseOf([0.25, -0.5])).action).toEqual({ act_probability: 0.25 }); + }); + + it("rounds the reported value to four decimals, as round(_, 4) does", () => { + expect(decodeRow(agent, caseOf([0.123456, 0.876544])).action).toEqual({ + act_probability: 0.1235, + }); + }); + + it("predict() normalises the provider's action logits once, at the model boundary", async () => { + const provider = { + async runEncoder(_b: unknown) { return { lastHidden: [[[1, 0]], [[0, 1]]] }; }, + async runHead(_h: unknown) { + return { logits: [[2.0, 0.0]], act: [[3.0, 0.0]] }; + }, + }; + const a = new Agent({ provider } as never); + const r = await a.systemOne("s", { + q: { type: "choice", instructions: "i", criteria: { a: "1", b: "2" } }, + } as never); + // softmax([3, 0])[0] = 0.95257…, one normalisation of the row the model exports as `act_logits`. + expect(r.answers.q.action).toEqual({ act_probability: 0.9526 }); + }); +});