Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions laya-ts/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -686,7 +686,13 @@ export class Agent extends HookRegistry {
for (let qi = 0; qi < ids.length; qi++) {
answers[ids[qi]] = this._decodeRow(
logits[row] as number[],
(act[row] as number[]) ?? [1, 0],
// The action head is normalised here, at the model boundary, as `laya/agent.py`'s
// `_forward` (agent.py:1329) and `laya/onnx_agent.py` (onnx_agent.py:805) do: the decode
// layer's contract -- the Java decoder's `actionProbabilities`, and the `act` rows in
// `laya-java/fixtures/decode.json` -- is a probability row. Softmaxing it a second time
// inside `_decodeRow` reported 0.6899 for the row `[0.9, 0.1]`, where the reference
// reports 0.9.
softmax((act[row] as number[]) ?? [1, 0]),
internals[qi],
built[s].items[qi].markers.length,
opts.lang ?? null,
Expand Down Expand Up @@ -720,7 +726,11 @@ export class Agent extends HookRegistry {
return out;
}

/** Decode one head row into the answer for its question. */
/**
* Decode one head row into the answer for its question. `actRow` is the model boundary's
* normalised action-probability row, so its first entry is the reported `act_probability` --
* exactly what `Agent._decode_answers` reads as `act[r, 0]`.
*/
private _decodeRow(
logitsRow: number[],
actRow: number[],
Expand All @@ -739,8 +749,9 @@ export class Agent extends HookRegistry {
: (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0);
const z = logitsRow.slice(0, k).map((v) => v / scale);
const p = softmax(z);
const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
const ext = { act_probability: r4(actP[0]) };
// Read, not re-normalised: the row arrives as probabilities. The `slice(0, Math.max(2, ...))`
// this replaces could only pad an Act row that validation had already refused for being empty.
const ext = { act_probability: r4(actRow[0]) };
// Same quantity on every question type (max(p)), so callers can gate across types on
// one number; `confidence` stays as-is for existing callers (entropy for choice/score).
// Histogram-binning recalibration (issue #871, Python parity) recalibrates ans_raw
Expand Down
152 changes: 152 additions & 0 deletions laya-ts/tests/decode-parity.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
import { describe, expect, it } from "vitest";
import { readFileSync } from "node:fs";
import { Agent } from "../src/agent.js";

// `laya-java/fixtures/decode.json` is the shared decode oracle: `gen_fixtures.py:1845-1860` hands
// each synthesised row to laya's own `Agent._decode_answers` and records the answer, so the same
// file is what the Java port asserts against field-by-field at tolerance 0.0
// (`laya-java/.../decode/DecoderParityTest.java`). The rows hit branches rather than look
// realistic -- every temperature bucket, a top-2 tie, an exactly uniform 20-option softmax, +/-700
// magnitudes that overflow a naive `exp`, a logits block three columns wider than the option
// count, and a per-language override -- which is the point: none of it needs a checkpoint.
//
// `option_order` is out of scope here. The port has no un-permutation step in the decode layer and
// no permutation step in the prompt layer, so the two permuted cases per checkpoint
// (`choice-4-permuted`, `score-permuted`) cannot agree on `choice`/`probabilities`/`score`. Their
// permutation-invariant fields are still asserted below, so the rest of the row stays covered.
const fixture = JSON.parse(
readFileSync(new URL("../../laya-java/fixtures/decode.json", import.meta.url), "utf8"),
) as Record<string, DecodeCheckpoint>;

type DecodeCase = {
question: { t: "choice" | "score" | "noul"; ins: string; crit?: unknown; labels?: unknown };
option_order: number[] | null;
lang: string | null;
logits: number[];
logits_block_width: number;
act: number[];
answer: Record<string, unknown>;
};

type DecodeCheckpoint = {
config: {
temperature: number[];
temperature_by_options: Record<string, number>;
lang_temperatures_raw: Record<
string,
{ temperature: number[]; temperature_by_options: Record<string, number> }
>;
};
qtypes: Record<string, number>;
cases: Record<string, DecodeCase>;
};

const fakeProvider = () => ({
async runEncoder(_b: unknown) { return { lastHidden: [[1, 0], [0, 1]] }; },
async runHead(_h: unknown) { return { logits: [[2, 0]], act: [[3, 0]] }; },
});

/** The Agent the reference's `shim` is: the checkpoint's clamped scales and one language override. */
function agentFor(ck: DecodeCheckpoint): Agent {
return new Agent({
provider: fakeProvider(),
cfg: {
temperature: ck.config.temperature,
temperature_by_options: ck.config.temperature_by_options,
},
lang_temperatures: ck.config.lang_temperatures_raw,
} as never);
}

/** The row as the head returns it: padded past the option count with large finite values. */
function paddedRow(c: DecodeCase): number[] {
const row = [...c.logits];
while (row.length < c.logits_block_width) row.push(99.0);
return row;
}

function decodeRow(agent: Agent, c: DecodeCase): Record<string, unknown> {
const decodeRow = (agent as unknown as {
_decodeRow(
logits: number[],
act: number[],
q: DecodeCase["question"],
k: number,
lang: string | null,
): Record<string, unknown>;
})._decodeRow;
return decodeRow.call(agent, paddedRow(c), c.act, c.question, c.logits.length, c.lang);
}

describe("decode.json parity", () => {
for (const [checkpointName, ck] of Object.entries(fixture)) {
if ("skipped" in (ck as unknown as Record<string, unknown>)) continue;
const agent = agentFor(ck);
describe(checkpointName, () => {
for (const [caseName, c] of Object.entries(ck.cases)) {
if (c.option_order !== null) continue;
it(`${caseName} decodes to the reference answer`, () => {
expect(decodeRow(agent, c)).toEqual(c.answer);
});
}
});
}
});

describe("decode.json parity across option_order", () => {
for (const [checkpointName, ck] of Object.entries(fixture)) {
if ("skipped" in (ck as unknown as Record<string, unknown>)) continue;
const agent = agentFor(ck);
for (const [caseName, c] of Object.entries(ck.cases)) {
if (c.option_order === null) continue;
it(`${checkpointName}/${caseName} keeps the permutation-invariant fields`, () => {
const got = decodeRow(agent, c);
expect(got.type).toBe(c.answer.type);
expect(got.confidence).toBe(c.answer.confidence);
expect(got.answer_confidence).toBe(c.answer.answer_confidence);
expect(got.action).toEqual(c.answer.action);
});
}
}
});

describe("the decode layer reads the action row as a probability", () => {
// `laya/agent.py:1329` (`_forward` returns `torch.softmax(act, -1)`) and `laya/onnx_agent.py:805`
// normalise the action head at the model boundary; `Agent._decode_answers` then reports
// `round(float(act[r, 0]), 4)` (`agent.py:1365`), and the Java decoder takes the pair as
// `actionProbabilities` (`DecoderParityTest.java:71-72`). Normalising inside the decode layer
// instead re-normalised the row: softmax([0.9, 0.1])[0] = 0.6899.
const ck = fixture.english;
const agent = agentFor(ck);
const caseOf = (act: number[]): DecodeCase => ({
...ck.cases["choice-2"],
act,
});

it("reports the first entry of the row the reference reports", () => {
expect(decodeRow(agent, caseOf([0.9, 0.1])).action).toEqual({ act_probability: 0.9 });
expect(decodeRow(agent, caseOf([0.99, 0.01])).action).toEqual({ act_probability: 0.99 });
expect(decodeRow(agent, caseOf([0.25, -0.5])).action).toEqual({ act_probability: 0.25 });
});

it("rounds the reported value to four decimals, as round(_, 4) does", () => {
expect(decodeRow(agent, caseOf([0.123456, 0.876544])).action).toEqual({
act_probability: 0.1235,
});
});

it("predict() normalises the provider's action logits once, at the model boundary", async () => {
const provider = {
async runEncoder(_b: unknown) { return { lastHidden: [[[1, 0]], [[0, 1]]] }; },
async runHead(_h: unknown) {
return { logits: [[2.0, 0.0]], act: [[3.0, 0.0]] };
},
};
const a = new Agent({ provider } as never);
const r = await a.systemOne("s", {
q: { type: "choice", instructions: "i", criteria: { a: "1", b: "2" } },
} as never);
// softmax([3, 0])[0] = 0.95257…, one normalisation of the row the model exports as `act_logits`.
expect(r.answers.q.action).toEqual({ act_probability: 0.9526 });
});
});
Loading