Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions laya-ts/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
clampTemperature,
collateItems,
confidenceFromProbs,
answerConfidence,
renderOptions,
sequenceWithState,
serializeState,
Expand Down Expand Up @@ -42,6 +43,7 @@ export interface ChoiceAnswer {
choice: string;
probabilities: Record<string, number>;
confidence: number;
answer_confidence: number;
action: ActionInfo;
}

Expand All @@ -51,13 +53,15 @@ export interface ScoreAnswer {
legend: Record<string, unknown>;
probabilities: Record<string, number>;
confidence: number;
answer_confidence: number;
action: ActionInfo;
}

export interface NoulAnswer {
type: "noul";
noul: number;
confidence: number;
answer_confidence: number;
action: ActionInfo;
}

Expand Down Expand Up @@ -399,6 +403,9 @@ export class Agent extends HookRegistry {
const actRow = (act[r] as number[]) ?? [1, 0];
const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
const ext = { act_probability: r4(actP[0]) };
// Same quantity on every question type (max(p)), so callers can gate across types on
// one number; `confidence` stays as-is for existing callers (entropy for choice/score).
const ansConf = r4(answerConfidence(p));
if (q.t === "choice") {
const keys = Object.keys(q.crit as Record<string, unknown>);
let best = 0;
Expand All @@ -408,6 +415,7 @@ export class Agent extends HookRegistry {
choice: keys[best],
probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),
confidence: r4(confidenceFromProbs(p)),
answer_confidence: ansConf,
action: ext,
};
} else if (q.t === "score") {
Expand All @@ -418,6 +426,7 @@ export class Agent extends HookRegistry {
legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])),
probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),
confidence: r4(confidenceFromProbs(p)),
answer_confidence: ansConf,
action: ext,
};
} else {
Expand All @@ -426,6 +435,8 @@ export class Agent extends HookRegistry {
type: "noul",
noul: r4(pt),
confidence: r4(Math.max(pt, 1 - pt)),
// over two options max(p_true, 1 - p_true) is max(p): identical to confidence here
answer_confidence: ansConf,
action: ext,
};
}
Expand Down
8 changes: 8 additions & 0 deletions laya-ts/src/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,14 @@ export function confidenceFromProbs(p: number[]): number {
const ent = -p.reduce((a, v) => a + v * Math.log(Math.max(v, 1e-12)), 0);
return Math.min(1, Math.max(0, 1 - ent / Math.log(k)));
}
export function answerConfidence(p: number[]): number {
// Probability mass on the reported answer: max(p). This is the quantity temperature
// scaling fits and the one every calibration figure is computed on, so it is stable
// across option counts and comparable across question types -- unlike
// confidenceFromProbs, whose entropy scale moves with k.
if (p.length < 1) return 1.0;
return Math.min(1, Math.max(0, Math.max(...p)));
}
export const TEMP_MIN = 0.5, TEMP_MAX = 5.0;
export function clampTemperature(t: unknown): number {
if (t === null || t === undefined || t === "" || typeof t === "boolean") return 1.0;
Expand Down
1 change: 1 addition & 0 deletions laya-ts/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ export {
sequenceWithState,
softmax,
confidenceFromProbs,
answerConfidence,
clampTemperature,
tempBucket,
collateItems,
Expand Down
56 changes: 56 additions & 0 deletions laya-ts/tests/answer-confidence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import { describe, expect, it } from "vitest";
import { answerConfidence, confidenceFromProbs } from "../src/common.js";
import { Agent } from "../src/agent.js";

const fakeProvider = (n: number) => ({
// n logits with a known argmax at 0; softmax gives a fixed, checkable distribution
async runEncoder(_b: any) { return { lastHidden: [[1, 0], [0, 1]] }; },
async runHead(_h: any) {
const row = Array.from({ length: n }, (_, i) => (i === 0 ? 2 : 0));
return { logits: [row], act: [[3, 0]] };
},
});

function probs(provider: any, a: Agent) { return a as any; }

describe("answerConfidence", () => {
it("is max(p) on the reported answer", () => {
expect(answerConfidence([0.25, 0.6, 0.15])).toBeCloseTo(0.6);
expect(answerConfidence([1.0])).toBe(1.0);
});

it("returns 1.0 for an empty distribution (Python parity: k < 1)", () => {
expect(answerConfidence([])).toBe(1.0);
});

it("stays at the same value across option counts, unlike the entropy confidence", () => {
// The #361 table: same top probability 0.9, different label-set sizes.
const fill = (k: number) => {
const rest = Array.from({ length: k - 1 }, () => 0.1 / (k - 1));
return [0.9, ...rest];
};
for (const k of [2, 5, 20, 77]) {
expect(answerConfidence(fill(k))).toBeCloseTo(0.9, 5);
}
// entropy confidence rises with k for the same max(p)
expect(confidenceFromProbs(fill(2))).toBeLessThan(confidenceFromProbs(fill(20)));
});

it("is attached to every answer type by the agent decode", async () => {
const mk = (n: number) => new Agent({ provider: fakeProvider(n) } as any);
const choice: any = await mk(3).systemOne("s", {
c: { type: "choice", instructions: "q?", criteria: { a: "x", b: "y", c: "z" } },
});
const top = Math.max(...Object.values(choice.answers.c.probabilities).map(Number));
expect(choice.answers.c.answer_confidence).toBeCloseTo(top, 4);

const score: any = await mk(3).systemOne("s", {
s: { type: "score", instructions: "q?", criteria: ["lo", "mid", "hi"] },
});
expect(score.answers.s.answer_confidence).toBeCloseTo(top, 4);

const noul: any = await mk(2).systemOne("s", { n: { type: "noul", instructions: "q?" } });
// over two options max(p) equals the existing noul confidence
expect(noul.answers.n.answer_confidence).toBeCloseTo(noul.answers.n.confidence, 4);
});
});