Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions laya-ts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,10 @@ const results = await agent.predictBatch(states, questions, { batchSize: 32 });
// in as few forward passes as possible. Results keep input order and carry `routing`.
const decisions = router.routeBatch(requests); // validate + route, nothing loads
const routed = await router.predictBatch(requests, 32); // == router.predictMany(...)
// Token budgets belong to each request; omitted/null values use the Agent config.
const budgeted = await router.predictBatch([
{ state: "ticket", questions, maxLen: 64, headMaxLen: 16 },
]);
// Per-call hooks are composed like predict: installed hooks first, then this list,
// then onPredictStart / onPredictEnd. null and [] add nothing. hooksRaise overrides
// the Router's policy for this call. The list runs once per request, including onRoute.
Expand All @@ -315,8 +319,8 @@ const traced = await router.predictBatch(requests, null, {
});

// requests routed to the same checkpoint still split into separate batches when their
// question schemas differ (order-sensitively), when per-request start hooks set
// different ctx.maxLen / ctx.headMaxLen overrides, or — for an agent carrying
// question schemas differ (order-sensitively), when their maxLen / headMaxLen
// budgets differ after start hooks (which can replace or clear them), or — for an agent carrying
// lang_temperatures — when their languages differ: each request's effective language (an
// explicit lang, otherwise the detected non-English one) is forwarded to the agent, so
// the batched path scores exactly like predict. Router-level predict hooks run once per
Expand Down
14 changes: 11 additions & 3 deletions laya-ts/src/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ export interface PredictBatchOptions {
hooksRaise?: boolean;
}

/** One item of a Router.predictBatch/routeBatch batch: state, questions, route overrides. */
/** One item of a Router.predictBatch/routeBatch batch: state, questions, route and budget overrides. */
export interface BatchRequest {
state: unknown;
questions: Record<string, QuestionDef>;
Expand All @@ -198,6 +198,9 @@ export interface BatchRequest {
lang?: string | null;
langGuess?: LangGuess;
lang_guess?: LangGuess;
/** Per-request token budgets; null = agent config. Start hooks may override them. */
maxLen?: number | null;
headMaxLen?: number | null;
}

function questionSchema(questions: Record<string, unknown>): string {
Expand Down Expand Up @@ -931,7 +934,7 @@ export class Router extends HookRegistry {
systemOne(
state: unknown,
questions: Record<string, QuestionDef>,
opts?: { lang?: string | null },
opts?: { lang?: string | null; maxLen?: number | null; headMaxLen?: number | null },
): Promise<SystemOneResult>;
langTemperatures?: Record<string, unknown>;
};
Expand All @@ -945,6 +948,8 @@ export class Router extends HookRegistry {
model: modelName,
agent,
router: this,
maxLen: requests[i].maxLen ?? null,
headMaxLen: requests[i].headMaxLen ?? null,
});
started.push(ctx);
dispatch(active, "onPredictStart", ctx, { raiseErrors });
Expand Down Expand Up @@ -1032,7 +1037,10 @@ export class Router extends HookRegistry {
// still work; their states simply share no forward pass.
batchResults = [];
for (const [, ctx] of group.items) {
const singleOpts = group.lang === null ? {} : { lang: group.lang };
const singleOpts: { lang?: string; maxLen?: number; headMaxLen?: number } =
group.lang === null ? {} : { lang: group.lang };
if (group.maxLen !== null) singleOpts.maxLen = group.maxLen;
if (group.headMaxLen !== null) singleOpts.headMaxLen = group.headMaxLen;
markDefaultsRan(singleOpts);
batchResults.push(
await agent.systemOne(
Expand Down
89 changes: 89 additions & 0 deletions laya-ts/tests/router-batch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,64 @@ describe("Router.routeBatch / predictBatch", () => {
expect(calls[1][3]).toEqual({ batchSize: null, maxLen: 64 });
});

it("seeds request budgets before hooks and groups by both final budgets", async () => {
const { router, calls } = makeRouter();
const seen: unknown[] = [];
const results = await router.predictBatch([
req("a", { model: "english", maxLen: 64, headMaxLen: 16 }),
req("b", { model: "english", maxLen: 128, headMaxLen: 16 }),
req("c", { model: "english", maxLen: 64, headMaxLen: 32 }),
req("d", { model: "english", maxLen: 64, headMaxLen: 16 }),
], 2, {
onPredictStart(ctx) { seen.push([ctx.maxLen, ctx.headMaxLen]); },
});
expect(seen).toEqual([[64, 16], [128, 16], [64, 32], [64, 16]]);
expect(calls.map((c) => [c[1], c[3]])).toEqual([
[["a", "d"], { batchSize: 2, maxLen: 64, headMaxLen: 16 }],
[["b"], { batchSize: 2, maxLen: 128, headMaxLen: 16 }],
[["c"], { batchSize: 2, maxLen: 64, headMaxLen: 32 }],
]);
expect(results.map((r) => r.answers.seen)).toEqual(["a", "b", "c", "d"]);
});

it("lets hooks replace or clear explicit budgets before grouping", async () => {
const { router, calls } = makeRouter();
const seen: unknown[] = [];
await router.predictBatch([
req("replace", { model: "english", maxLen: 64, headMaxLen: 16 }),
req("clear", { model: "english", maxLen: 128, headMaxLen: 32 }),
req("defaults", { model: "english" }),
], null, {
onPredictStart(ctx) {
seen.push([ctx.maxLen, ctx.headMaxLen]);
if (ctx.states[0] === "replace") {
ctx.maxLen = 96;
ctx.headMaxLen = 24;
} else if (ctx.states[0] === "clear") {
ctx.maxLen = ctx.headMaxLen = null;
}
},
});
expect(seen).toEqual([[64, 16], [128, 32], [null, null]]);
expect(calls.map((c) => [c[1], c[3]])).toEqual([
[["replace"], { batchSize: null, maxLen: 96, headMaxLen: 24 }],
[["clear", "defaults"], { batchSize: null }],
]);
});

it("keeps absent and null budgets unset without dropping zero overrides", async () => {
const { router, calls } = makeRouter();
await router.predictBatch([
req("absent", { model: "english" }),
req("null", { model: "english", maxLen: null, headMaxLen: null }),
req("zero", { model: "english", maxLen: 0, headMaxLen: 0 }),
]);
expect(calls.map((c) => [c[1], c[3]])).toEqual([
[["absent", "null"], { batchSize: null }],
[["zero"], { batchSize: null, maxLen: 0, headMaxLen: 0 }],
]);
});

it("raises when the agent returns the wrong number of results", async () => {
const router = new Router({
loader: (() => ({
Expand All @@ -510,6 +568,37 @@ describe("Router.routeBatch / predictBatch", () => {
).rejects.toThrow(/returned 0 results for 2 states/);
});

it.each([false, true])("forwards fallback budgets with hook override=%s", async (override) => {
const calls: unknown[] = [];
const router = new Router({
loader: (() => ({
langTemperatures: { en: {} },
async systemOne(state: unknown, _questions: unknown, opts: unknown) {
calls.push([state, opts]);
return { model: "stub", answers: { seen: state }, usage: {} };
},
})) as never,
});
const results = await router.predictBatch([
req("budgeted", { model: "english", lang: "en", maxLen: 64, headMaxLen: 16 }),
req("defaults", { model: "english", lang: "en", maxLen: null, headMaxLen: null }),
req("zero", { model: "english", lang: "en", maxLen: 0, headMaxLen: 0 }),
], 2, {
onPredictStart(ctx) {
if (override && ctx.states[0] === "budgeted") {
ctx.maxLen = 96;
ctx.headMaxLen = 24;
}
},
});
expect(calls).toEqual([
["budgeted", { lang: "en", maxLen: override ? 96 : 64, headMaxLen: override ? 24 : 16 }],
["defaults", { lang: "en" }],
["zero", { lang: "en", maxLen: 0, headMaxLen: 0 }],
]);
expect(results.map((r) => r.answers.seen)).toEqual(["budgeted", "defaults", "zero"]);
});

it("falls back to systemOne for agent-like objects without predictBatch", async () => {
const router = new Router({
loader: (() => ({
Expand Down
34 changes: 34 additions & 0 deletions laya-ts/tests/structured.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,40 @@ describe("structured/decideBatch", () => {



describe("structured/decideBatch Router token budgets", () => {
it.each([32, 512])("matches single encoding when overriding the Agent default of %s", async (maxLen) => {
const encoderLengths: number[][] = [];
const agent = new Agent({
max_len: maxLen,
provider: {
async runEncoder(batch) {
encoderLengths.push(batch.inputIds.map((row) => row.length));
return { lastHidden: batch.inputIds.map(() => [[1, 0], [0, 1]]) };
},
async runHead(_hidden, batch) {
return {
logits: batch.inputIds.map(() => [2, 0]),
act: batch.inputIds.map(() => [3, 0]),
};
},
},
});
const router = new Router({ default: "english", loader: () => agent });
const schema = {
type: "object",
properties: { relevant: { type: "boolean", description: "Is the request relevant?" } },
};
const state = Array.from({ length: 160 }, (_, i) => `word${i}`).join(" ");
const opts = { model: "english", maxLen: 64, headMaxLen: 16, returnDetails: true as const };
const single = await router.decide(state, schema, opts);
const batch = await router.decideBatch([state, state], schema, opts);
expect(single.usage).toMatchObject({ input_tokens: 64, state_tokens_dropped: 114, truncated: true });
expect(batch.map((result) => result.usage)).toEqual([single.usage, single.usage]);
expect(batch.map((result) => result.values)).toEqual([single.values, single.values]);
expect(encoderLengths).toEqual([[64], [64, 64]]);
});
});

// `DecideBatchOptions` documents that anything it does not read itself is forwarded to the
// runner, hooks included. On a Router-like runner the call used to spread those options into the
// request objects and pass no third argument, so `Router.predictBatch` -- which reads hooks from
Expand Down
10 changes: 10 additions & 0 deletions laya-ts/tests/types/router-batch-budgets.types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import type { BatchRequest } from "../../src/index.js";

export const requests: BatchRequest[] = [
{ state: "ticket", questions: {}, maxLen: 64, headMaxLen: 16 },
{ state: "defaults", questions: {}, maxLen: null, headMaxLen: null },
{ state: "omitted", questions: {} },
];

// @ts-expect-error token budgets must be numbers or null
export const invalid: BatchRequest = { state: "ticket", questions: {}, maxLen: "64" };
Loading