Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 64 additions & 11 deletions laya-ts/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@ import {
serializeState,
softmax,
tempBucket,
unpermuteProbs,
} from "./common.js";
import type { BinningMap, MinConfidence, OptionStats, SequenceStats } from "./common.js";
import type { BinningMap, InternalQ, MinConfidence, OptionStats, SequenceStats } from "./common.js";
import type { Batch, SessionProvider } from "./providers.js";
import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";
import { decide, type DecideOptions, type DecisionResult } from "./structured.js";
Expand Down Expand Up @@ -170,6 +171,15 @@ function qidStr(qid: string): string {
return JSON.stringify(qid);
}

/** How many options a validated question definition renders to -- `laya.agent._option_count`
* (`laya/agent.py:321-330`): a choice has one per criterion, a score one per level, a noul is always
* the pair [false, true]. */
function optionCount(qdef: Record<string, unknown>): number {
if (qdef["type"] === "noul") return 2;
const crit = qdef["criteria"];
return Array.isArray(crit) || (typeof crit === "object" && crit !== null) ? Object.keys(crit).length : 0;
}

export function checkQuestion(qid: string, qdef: unknown): void {
if (typeof qid !== "string" || !qid.trim()) {
throw new Error(`question id must be a non-empty string, got ${qidStr(qid)}`);
Expand Down Expand Up @@ -290,6 +300,29 @@ export function checkQuestion(qid: string, qdef: unknown): void {
);
}
}
// Slot `s` of the head's row shows option `order[s]`, so an order that is not a permutation of the
// option indices would either drop an option or show one twice. Python refuses it here
// (`laya/agent.py:1088-1099`) rather than let it reach the encoder, and the same numbers say why:
// the decoder un-permutes by the same list, so a short order leaves slots unmapped.
if (Object.hasOwn(q, "option_order")) {
const order = q["option_order"];
const n = optionCount(q);
const ints = Array.isArray(order)
// Python keeps `isinstance(i, int) and not isinstance(i, bool)`, which rejects a float option
// index; JavaScript has one number type, so an integral value is the closest match, and
// `[1.0, 0.0]` is not a value a caller can even write here.
? order.filter((i) => typeof i === "number" && Number.isInteger(i))
: null;
const isPermutation =
ints !== null && ints.length === n &&
[...ints].sort((a, b) => a - b).join(",") === Array.from({ length: n }, (_, i) => i).join(",");
if (!isPermutation) {
throw new Error(
`question ${qidStr(qid)}: 'option_order' must be a permutation of range(${n}) -- one slot per ` +
`option, each option once -- got ${JSON.stringify(order)}`,
);
}
}
if ("labels" in q && t !== "noul") {
throw new Error(`question ${qidStr(qid)}: 'labels' is only supported for noul questions`);
}
Expand Down Expand Up @@ -332,7 +365,7 @@ function isEmptyOverride(v: unknown): boolean {
return Array.isArray(v) && v.length === 0; // [] -- truthy in JavaScript
}

export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } {
export function toInternal(qdef: QuestionDef): InternalQ {
const t = qdef["type"] as "choice" | "score" | "noul";
let crit: unknown = qdef["criteria"];
if (t === "choice" && Array.isArray(crit)) {
Expand All @@ -342,13 +375,19 @@ export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul";
}
let ins: unknown = qdef["instructions"];
if (typeof ins !== "string") ins = serializeState(ins);
const out: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } = {
const out: InternalQ = {
t, ins: ins as string, crit,
};
if (t === "noul" && qdef["labels"] && typeof qdef["labels"] === "object" && !Array.isArray(qdef["labels"])) {
const labels = qdef["labels"] as Record<string, unknown>;
out.labels = { false: String(labels["false"]).trim(), true: String(labels["true"]).trim() };
}
// `Agent._to_internal` carries the validated order onto the internal question (`laya/agent.py:1128-1129`),
// which is what lets one object drive both halves: `_encodeItems` presents options in it and
// `_decodeRow` inverts the row back out of it.
if (Object.hasOwn(qdef, "option_order")) {
out.optionOrder = (qdef["option_order"] as number[]).map((i) => Math.trunc(Number(i)));
}
return out;
}

Expand Down Expand Up @@ -590,7 +629,7 @@ export class Agent extends HookRegistry {
private _encodeItems(
state: unknown,
ids: string[],
internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[],
internals: InternalQ[],
maxLen: number,
headMaxLen: number,
): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number; stats: SequenceStats[] } {
Expand All @@ -602,7 +641,7 @@ export class Agent extends HookRegistry {
for (let qi = 0; qi < ids.length; qi++) {
const qid = ids[qi];
const q = internals[qi];
const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen);
const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen, q.optionOrder);
const seq = sequenceWithState(
prefix,
stAll,
Expand Down Expand Up @@ -686,7 +725,13 @@ export class Agent extends HookRegistry {
for (let qi = 0; qi < ids.length; qi++) {
answers[ids[qi]] = this._decodeRow(
logits[row] as number[],
(act[row] as number[]) ?? [1, 0],
// The action head is normalised here, at the model boundary, as `laya/agent.py`'s
// `_forward` (agent.py:1329) and `laya/onnx_agent.py` (onnx_agent.py:805) do: the decode
// layer's contract -- the Java decoder's `actionProbabilities`, and the `act` rows in
// `laya-java/fixtures/decode.json` -- is a probability row. Softmaxing it a second time
// inside `_decodeRow` reported 0.6899 for the row `[0.9, 0.1]`, where the reference
// reports 0.9.
softmax((act[row] as number[]) ?? [1, 0]),
internals[qi],
built[s].items[qi].markers.length,
opts.lang ?? null,
Expand Down Expand Up @@ -720,11 +765,15 @@ export class Agent extends HookRegistry {
return out;
}

/** Decode one head row into the answer for its question. */
/**
* Decode one head row into the answer for its question. `actRow` is the model boundary's
* normalised action-probability row, so its first entry is the reported `act_probability` --
* exactly what `Agent._decode_answers` reads as `act[r, 0]`.
*/
private _decodeRow(
logitsRow: number[],
actRow: number[],
q: { t: "choice" | "score" | "noul"; ins: string; crit: unknown },
q: InternalQ,
k: number,
lang: string | null = null,
): SystemAnswer {
Expand All @@ -738,9 +787,13 @@ export class Agent extends HookRegistry {
? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0)
: (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0);
const z = logitsRow.slice(0, k).map((v) => v / scale);
const p = softmax(z);
const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
const ext = { act_probability: r4(actP[0]) };
// The row comes back in slot order and everything below indexes by option, so invert it
// here -- `p.argmax()`/`keys[best]`, `i * p[i]` and `p[1]` are all option-indexed. Same
// position as `laya/agent.py:1350`, immediately after the softmax.
const p = unpermuteProbs(softmax(z), q.optionOrder);
// Read, not re-normalised: the row arrives as probabilities. The `slice(0, Math.max(2, ...))`
// this replaces could only pad an Act row that validation had already refused for being empty.
const ext = { act_probability: r4(actRow[0]) };
// Same quantity on every question type (max(p)), so callers can gate across types on
// one number; `confidence` stays as-is for existing callers (entropy for choice/score).
// Histogram-binning recalibration (issue #871, Python parity) recalibrates ans_raw
Expand Down
24 changes: 23 additions & 1 deletion laya-ts/src/common.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
import type { TokenizerLike } from "./tokenizer.js";
export type QType = "choice" | "score" | "noul";
export interface InternalQ { t: QType; ins: string; crit: unknown; labels?: { false: string; true: string } }
export interface InternalQ {
t: QType;
ins: string;
crit: unknown;
labels?: { false: string; true: string };
/** The caller's option indices in the order the head should see them: slot `s` shows option
`optionOrder[s]`. Set by `toInternal` from a question's `option_order`. */
optionOrder?: number[];
}
/** Python `json.dumps` for a finite number. JavaScript prints the same shortest round-trip
digits, but Python switches to exponent notation below 1e-4 (JavaScript: below 1e-6) and pads the
exponent to two digits (`5e-05`, not `5e-5`). An integer-valued number prints as before, which
Expand Down Expand Up @@ -197,6 +205,20 @@ export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,
const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));
return sequenceWithState(buildQuestionPrefix(tok, q, maxLen, headMaxLen, optionOrder), stAll, tok.sepId, maxLen, truncateLeft);
}
/** Put a slot-ordered probability row back into the caller's option order.
*
* `buildQuestionPrefix` puts option `order[s]` in slot `s`, so a head row comes back indexed by
* slot, while everything downstream indexes by option -- `keys[best]` for a choice, `i * p[i]` for a
* score level, `p[1]` for noul-true. Without the inverse the probabilities stay attached to the
* wrong options, which is silent. Mirrors `laya.common.unpermute_probs` (`laya/common.py:557`): a
* missing order, or one whose length is not the row's, returns the row untouched, so the canonical
* path is unaffected. */
export function unpermuteProbs(p: number[], order?: number[] | null): number[] {
if (!order || order.length !== p.length) return p;
const canonical = new Array<number>(p.length);
for (let s = 0; s < order.length; s++) canonical[order[s]] = p[s];
return canonical;
}
export function softmax(z: number[]): number[] {
const m = Math.max(...z);
const e = z.map((v) => Math.exp(v - m));
Expand Down
52 changes: 37 additions & 15 deletions laya-ts/src/shortlist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,34 @@ function subsetCriteria(criteria: unknown, labels: string[]): unknown {
return Object.fromEntries(labels.map((l) => [l, (criteria as Record<string, unknown>)[l]]));
}

/** Return `question` without an `option_order` that no longer fits it.
*
* `option_order` holds one slot per option, so an order that fitted the caller's label set does not
* fit a narrowed one -- and once the agent reads the field at all, that mismatch is a refused
* question, not a silently ignored one. Remapping the order onto the kept labels would move options
* between slots, so it is dropped instead, once per question per call. Mirrors
* `laya.shortlist._drop_stale_order` (`laya/shortlist.py:428`), which both `predict_shortlist` and
* `predict_tournament` run on every narrowed choice; a passthrough keeps its order. */
function dropStaleOrder(qid: string, question: Record<string, unknown>, nOptions: number,
warned: Set<string>): Record<string, unknown> {
if (!Object.hasOwn(question, "option_order")) return question;
const out = { ...question };
const order = out["option_order"];
delete out["option_order"];
if (!warned.has(qid)) {
warned.add(qid);
// The same shape as the reference's warning: the slot count the order carried, so a caller can
// see how much of the label set the narrowing dropped.
const had = Array.isArray(order) ? `${order.length} slots` : JSON.stringify(order);
console.warn(
`laya.shortlist: question ${JSON.stringify(qid)}: 'option_order' (${had}) was dropped because the ` +
`label set was narrowed to ${nOptions}; the order no longer fits the kept options and remapping ` +
`it onto them would move options between slots.`,
);
}
return out;
}

/** Return the top-k choice labels for state. Skips embed_fn when k >= n. */
export async function shortlistChoice(
state: unknown,
Expand Down Expand Up @@ -172,6 +200,7 @@ export async function predictShortlist(
const embed = embedFn ?? (needsEmbed ? embedFnFromAgent(agent) : null);
const reduced: Record<string, unknown> = {};
const meta: Record<string, ShortlistMeta> = {};
const warned = new Set<string>();
for (const [qid, qdef] of Object.entries(questions)) {
if (typeof qdef !== "object" || qdef === null || Array.isArray(qdef) || (qdef as Record<string, unknown>)["type"] !== "choice") {
reduced[qid] = qdef;
Expand All @@ -187,7 +216,8 @@ export async function predictShortlist(
reduced[qid] = qdef;
continue;
}
reduced[qid] = { ...q, criteria: subsetCriteria(q["criteria"], labels) };
reduced[qid] = dropStaleOrder(
qid, { ...q, criteria: subsetCriteria(q["criteria"], labels) }, labels.length, warned);
}
const a = agent as Record<string, unknown>;
const fn = a?.["predict"] ?? a?.["systemOne"] ?? a?.["system_one"];
Expand Down Expand Up @@ -248,20 +278,12 @@ export async function predictTournament(
return result as SystemOneResult;
};
const warned = new Set<string>();
const cut = (qid: string, labels: string[]): QuestionDef => {
const original = questions[qid];
const narrowed: QuestionDef = { ...original, criteria: subsetCriteria(original["criteria"], labels) };
if (Object.hasOwn(narrowed, "option_order")) {
delete narrowed["option_order"];
if (!warned.has(qid)) {
warned.add(qid);
console.warn(
`laya.shortlist: question ${JSON.stringify(qid)}: 'option_order' was dropped because the label set was narrowed to ${labels.length}; the order no longer fits the kept options.`,
);
}
}
return narrowed;
};
const cut = (qid: string, labels: string[]): QuestionDef => dropStaleOrder(
qid,
{ ...questions[qid], criteria: subsetCriteria(questions[qid]["criteria"], labels) },
labels.length,
warned,
) as QuestionDef;

while (true) {
const groups: { qid: string; labels: string[] }[] = [];
Expand Down
Loading
Loading