diff --git a/laya-ts/README.md b/laya-ts/README.md index a64f9be2b..011721ef5 100644 --- a/laya-ts/README.md +++ b/laya-ts/README.md @@ -96,7 +96,42 @@ instead of a schema to get raw answers. Zod/TypeBox users can pass `z.toJSONSche any object with a `toJSONSchema()` method is accepted. `planFromJsonSchema`, `questionsFromJsonSchema` and `answersToJson` expose the planning and projection steps. -## Shortlist (many labels) +## Batch prediction + +Batch prediction evaluates multiple states together using shared forward passes. Results preserve the exact input order. + +### Agent.predictBatch + +`agent.predictBatch(states, questions, opts?)` answers questions across multiple states in a single forward pass, chunking states when `batchSize` is set in options: + +```ts +const states = [ + { body: "charged twice, refund please" }, + { body: "cannot log into my account" }, +]; + +const results = await agent.predictBatch(states, { + intent: { type: "choice", instructions: "What does the customer want?", criteria: { refund: "money back", other: "anything else" } }, +}, { batchSize: 32 }); + +console.log(results[0].answers.intent, results[1].answers.intent); +``` + +### Router.predictBatch and Router.routeBatch + +`router.routeBatch(requests)` resolves routing decisions for heterogeneous requests without loading model checkpoints. `router.predictBatch(requests, opts?)` groups requests by routed checkpoint and question schema to share forward passes: + +```ts +const requests = [ + { state: { body: "charged twice" }, questions: { intent: /* ... */ } }, + { state: { body: "मुझसे दो बार शुल्क लिया गया" }, questions: { intent: /* ... */ } }, +]; + +const decisions = router.routeBatch(requests); +const results = await router.predictBatch(requests, { batchSize: 32 }); +// results[i].routing records the RouteDecision for requests[i] +``` + ## Shortlist (many labels) ```ts diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts index 246ff0f67..58482ab70 100644 --- a/laya-ts/src/agent.ts +++ b/laya-ts/src/agent.ts @@ -11,6 +11,8 @@ import { serializeState, softmax, tempBucket, + type CollateItem, + type QuestionPrefix, } from "./common.js"; import type { Batch, SessionProvider } from "./providers.js"; import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js"; @@ -124,6 +126,15 @@ export interface PredictOptions { onPredictStart?: PredictHook; onPredictEnd?: PredictHook; hooksRaise?: boolean; + maxLen?: number; + max_len?: number; + headMaxLen?: number; + head_max_len?: number; +} + +export interface PredictBatchOptions extends PredictOptions { + batchSize?: number; + batch_size?: number; } function qidStr(qid: string): string { @@ -348,36 +359,81 @@ export class Agent extends HookRegistry { questions: Record, opts: PredictOptions = {}, ): Promise { - return (await this._predictHooked([state], questions, opts))[0]; + return (await this.predictBatch([state], questions, opts))[0]; + } + + async predict( + state: unknown, + questions: Record, + opts: PredictOptions = {}, + ): Promise { + return this.systemOne(state, questions, opts); } - private async _predictHooked( + /** + * Evaluate typed questions across many states, packing them into shared forward passes. + * + * `opts.batchSize` bounds the number of states per forward pass; without it, all states + * share one forward pass. + * + * Input ordering is strictly preserved. + */ + async predictBatch( states: unknown[], questions: Record, - opts: PredictOptions, + opts: PredictBatchOptions = {}, ): Promise { const active = composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd); const raiseErrors = opts.hooksRaise ?? this.hooksRaise; + const maxLen = opts.maxLen ?? opts.max_len ?? null; + const headMaxLen = opts.headMaxLen ?? opts.head_max_len ?? null; const ctx = new PredictContext({ states, questions: questions as Record, agent: this, + maxLen, + headMaxLen, }); try { dispatch(active, "onPredictStart", ctx, { raiseErrors }); if (ctx.results === null) { - const out: SystemOneResult[] = []; - for (const st of ctx.states) { - out.push( - await this._systemOneCore( - st, - ctx.questions as Record, - opts.lang ?? null, - ), + if (!Array.isArray(ctx.states)) { + throw new TypeError( + "predictBatch expects an array of states; pass a single state to predict()/systemOne().", ); } - ctx.results = out as unknown as Record[]; - ctx.model ??= out[0]?.model ?? null; + const rawBatchSize = opts.batchSize ?? opts.batch_size; + if (rawBatchSize !== undefined && (!Number.isInteger(rawBatchSize) || (rawBatchSize as number) < 1)) { + throw new Error(`batchSize must be a positive integer, got ${JSON.stringify(rawBatchSize)}`); + } + const effectiveStates = ctx.states; + if (effectiveStates.length === 0) { + ctx.results = []; + } else { + const effectiveQuestions = ctx.questions as Record; + const ids = Object.keys(effectiveQuestions ?? {}); + if (ids.length === 0) { + ctx.results = effectiveStates.map(() => ({ + model: "laya-rl-agent", + answers: {}, + usage: { input_tokens: 0, output_tokens: 0 }, + })); + ctx.model ??= "laya-rl-agent"; + } else { + const chunk = rawBatchSize && rawBatchSize > 0 ? (rawBatchSize as number) : effectiveStates.length; + const out = await this._predictBatchCore( + effectiveStates, + effectiveQuestions, + ids, + chunk, + ctx.maxLen, + ctx.headMaxLen, + opts.lang ?? null, + ); + ctx.results = out as unknown as Record[]; + ctx.model ??= (ctx.results[0] as unknown as SystemOneResult)?.model ?? null; + } + } } } catch (err) { ctx.error = err; @@ -400,116 +456,168 @@ export class Agent extends HookRegistry { return ctx.results as unknown as SystemOneResult[]; } - private async _systemOneCore( - state: unknown, + private async _predictBatchCore( + states: unknown[], questions: Record, + ids: string[], + chunk: number, + overrideMaxLen: number | null, + overrideHeadMaxLen: number | null, lang: string | null = null, - ): Promise { - const ids = Object.keys(questions ?? {}); - if (ids.length === 0) { - return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } }; - } - const items: { ids: number[]; markers: number[]; qtype: number }[] = []; - const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }[] = []; - // The state text is shared by every question and is usually the longest text in the - // sequence — encode it once and compose the per-question prefixes onto it. - const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" ")); + ): Promise { + const effectiveMaxLen = overrideMaxLen ?? this.maxLen; + const effectiveHeadMaxLen = overrideHeadMaxLen ?? this.headMaxLen; + + const questionData: Array<{ + qid: string; + q: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }; + prefix: QuestionPrefix; + expectedMarkers: number; + }> = []; + for (const qid of ids) { checkQuestion(qid, questions[qid]); const q = toInternal(questions[qid]); - internals.push(q); - const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen); - const { ids: seq, markers } = sequenceWithState( - prefix, stAll, this.tok.sepId, this.maxLen, Array.isArray(state), - ); - if (markers.length !== renderOptions(q).length) { - throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`); - } - items.push({ ids: seq, markers, qtype: QTYPES[q.t] }); - } - const collated = collateItems([items], this.tok.padId); - if (!collated) throw new Error("no items to collate"); - const batch: Batch = collated; - const nTokens = batch.attentionMask.flat().reduce((a, b) => a + b, 0); - const { lastHidden } = await this.provider.runEncoder(batch); - const { logits, act } = await this.provider.runHead(lastHidden, batch); - if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < ids.length || act.length < ids.length) { - throw new Error("model provider returned fewer output rows than input items"); + const prefix = buildQuestionPrefix(this.tok, q, effectiveMaxLen, effectiveHeadMaxLen); + questionData.push({ + qid, + q, + prefix, + expectedMarkers: renderOptions(q).length, + }); } - for (let r = 0; r < ids.length; r++) { - const k = items[r].markers.length; - if (!Array.isArray(logits[r]) || logits[r].length < k || !Array.isArray(act[r]) || act[r].length === 0 || - logits[r].some((v) => !Number.isFinite(v)) || act[r].some((v) => !Number.isFinite(v))) { - throw new Error(`model provider returned invalid output for question ${ids[r]}`); + + const results: SystemOneResult[] = []; + + for (let start = 0; start < states.length; start += chunk) { + const chunkStates = states.slice(start, start + chunk); + const chunkStateItems: CollateItem[][] = []; + + for (const st of chunkStates) { + const stAll = this.tok.encode(serializeState(st).split(this.tok.maskToken).join(" ")); + const stateItems: CollateItem[] = []; + for (const { qid, q, prefix, expectedMarkers } of questionData) { + const { ids: seq, markers } = sequenceWithState( + prefix, + stAll, + this.tok.sepId, + effectiveMaxLen, + Array.isArray(st), + ); + if (markers.length !== expectedMarkers) { + throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${effectiveHeadMaxLen}`); + } + stateItems.push({ ids: seq, markers, qtype: QTYPES[q.t] }); + } + chunkStateItems.push(stateItems); } - } - const answers: Record = {}; - for (let r = 0; r < ids.length; r++) { - const qid = ids[r]; - const q = internals[r]; - const k = items[r].markers.length; - const qt = QTYPES[q.t]; - const bucket = tempBucket(qt, k); - // Python parity (_decode_answers): a matching lang override replaces the scale - // wholesale — its own temperature_by_options first, then its 3-slot temperature — - // so an override without buckets intentionally ignores the base per-bucket entries. - const langCfg = lang ? this.langTemperatures[lang.split("-")[0].toLowerCase()] : undefined; - const scale = langCfg - ? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0) - : (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0); - const z = (logits[r] as number[]).slice(0, k).map((v) => v / scale); - const p = softmax(z); - const actRow = (act[r] as number[]) ?? [1, 0]; - const actP = softmax(actRow.slice(0, Math.max(2, actRow.length))); - const ext = { act_probability: r4(actP[0]) }; - // Same quantity on every question type (max(p)), so callers can gate across types on - // one number; `confidence` stays as-is for existing callers (entropy for choice/score). - const ansConf = r4(answerConfidence(p)); - if (q.t === "choice") { - const keys = Object.keys(q.crit as Record); - let best = 0; - for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i; - answers[qid] = { - type: "choice", - choice: keys[best], - probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])), - confidence: r4(confidenceFromProbs(p)), - answer_confidence: ansConf, - action: ext, - }; - } else if (q.t === "score") { - const exp = p.reduce((a, v, i) => a + i * v, 0); - answers[qid] = { - type: "score", - score: r4(exp), - legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])), - probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])), - confidence: r4(confidenceFromProbs(p)), - answer_confidence: ansConf, - action: ext, - }; - } else { - const pt = p[1] ?? 0; - answers[qid] = { - type: "noul", - noul: r4(pt), - confidence: r4(Math.max(pt, 1 - pt)), - // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here - answer_confidence: ansConf, - action: ext, - }; + const collated = collateItems(chunkStateItems, this.tok.padId); + if (!collated) throw new Error("no items to collate"); + const batch: Batch = collated; + const { lastHidden } = await this.provider.runEncoder(batch); + const { logits, act } = await this.provider.runHead(lastHidden, batch); + + const totalRows = chunkStateItems.reduce((acc, items) => acc + items.length, 0); + if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < totalRows || act.length < totalRows) { + throw new Error("model provider returned fewer output rows than input items"); + } + + let rowOffset = 0; + for (let sIdx = 0; sIdx < chunkStates.length; sIdx++) { + const stateItems = chunkStateItems[sIdx]; + const nRows = stateItems.length; + + for (let r = 0; r < nRows; r++) { + const globalRow = rowOffset + r; + const k = stateItems[r].markers.length; + if ( + !Array.isArray(logits[globalRow]) || + logits[globalRow].length < k || + !Array.isArray(act[globalRow]) || + act[globalRow].length === 0 || + logits[globalRow].some((v) => !Number.isFinite(v)) || + act[globalRow].some((v) => !Number.isFinite(v)) + ) { + throw new Error(`model provider returned invalid output for question ${qidStr(questionData[r].qid)}`); + } + } + + const answers: Record = {}; + for (let r = 0; r < nRows; r++) { + const globalRow = rowOffset + r; + const { qid, q } = questionData[r]; + const k = stateItems[r].markers.length; + const qt = QTYPES[q.t]; + const bucket = tempBucket(qt, k); + // Python parity (_decode_answers): a matching lang override replaces the scale + // wholesale — its own temperature_by_options first, then its 3-slot temperature — + // so an override without buckets intentionally ignores the base per-bucket entries. + const langCfg = lang ? this.langTemperatures[lang.split("-")[0].toLowerCase()] : undefined; + const scale = langCfg + ? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0) + : (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0); + const z = (logits[globalRow] as number[]).slice(0, k).map((v) => v / scale); + const p = softmax(z); + const actRow = (act[globalRow] as number[]) ?? [1, 0]; + const actP = softmax(actRow.slice(0, Math.max(2, actRow.length))); + const ext = { act_probability: r4(actP[0]) }; + // Same quantity on every question type (max(p)), so callers can gate across types on + // one number; `confidence` stays as-is for existing callers (entropy for choice/score). + const ansConf = r4(answerConfidence(p)); + + if (q.t === "choice") { + const keys = Object.keys(q.crit as Record); + let best = 0; + for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i; + answers[qid] = { + type: "choice", + choice: keys[best], + probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])), + confidence: r4(confidenceFromProbs(p)), + answer_confidence: ansConf, + action: ext, + }; + } else if (q.t === "score") { + const exp = p.reduce((a, v, i) => a + i * v, 0); + answers[qid] = { + type: "score", + score: r4(exp), + legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])), + probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])), + confidence: r4(confidenceFromProbs(p)), + answer_confidence: ansConf, + action: ext, + }; + } else { + const pt = p[1] ?? 0; + answers[qid] = { + type: "noul", + noul: r4(pt), + confidence: r4(Math.max(pt, 1 - pt)), + // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here + answer_confidence: ansConf, + action: ext, + }; + } + } + + let stateTokens = 0; + for (let r = 0; r < nRows; r++) { + stateTokens += batch.attentionMask[rowOffset + r].reduce((a, b) => a + b, 0); + } + + results.push({ + model: "laya-rl-agent", + answers, + usage: { input_tokens: stateTokens, output_tokens: 0 }, + }); + + rowOffset += nRows; } } - return { model: "laya-rl-agent", answers, usage: { input_tokens: nTokens, output_tokens: 0 } }; - } - async predict( - state: unknown, - questions: Record, - opts: PredictOptions = {}, - ): Promise { - return this.systemOne(state, questions, opts); + return results; } /** diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts index 80bd718e1..680b5988d 100644 --- a/laya-ts/src/index.ts +++ b/laya-ts/src/index.ts @@ -12,11 +12,12 @@ export type { AgentCfg, AgentOptions, PredictOptions, + PredictBatchOptions, } from "./agent.js"; export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle, loadWebBundle } from "./providers.js"; export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js"; export { Router, normaliseName, DEFAULT_MODELS } from "./router.js"; -export type { RoutedResult, RouteDecision, ModelName, ModelSpec } from "./router.js"; +export type { RoutedResult, RouteDecision, ModelName, ModelSpec, RouterBatchRequest, RouterBatchOptions } from "./router.js"; export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K } from "./shortlist.js"; export type { EmbedFn, ShortlistMeta } from "./shortlist.js"; export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js"; diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts index 6153103e6..54e81ae72 100644 --- a/laya-ts/src/router.ts +++ b/laya-ts/src/router.ts @@ -8,6 +8,7 @@ import { composeHooks, dispatch, normaliseHooks, + type Hook, type HookArg, type PredictHook, } from "./hooks.js"; @@ -136,6 +137,37 @@ export interface RouteOptions { hooksRaise?: boolean; } +export interface RouterBatchRequest { + state: unknown; + questions: Record; + model?: string | null; + task?: string | null; + lang?: string | null; + langGuess?: LangGuess; + lang_guess?: LangGuess; +} + +export interface RouterBatchOptions { + batchSize?: number | null; + batch_size?: number | null; +} + +export function _questionSchema(questions: Record): string { + return JSON.stringify(questions, (_key, val) => { + if (typeof val === "bigint" || typeof val === "symbol") return String(val); + if ( + val !== null && + typeof val === "object" && + !Array.isArray(val) && + val.constructor && + val.constructor.name !== "Object" + ) { + return String(val); + } + return val; + }); +} + function toSpec(spec: string | ModelSpec | [string, string | null]): ModelSpec { if (typeof spec === "string") return { repo: spec, subfolder: null }; if (Array.isArray(spec)) { @@ -532,4 +564,278 @@ export class Router extends HookRegistry { ): Promise { return this.predict(state, questions, opts); } + + routeBatch(requests: RouterBatchRequest[]): RouteDecision[] { + if (!Array.isArray(requests)) { + throw new TypeError("requests must be a sequence of request dictionaries"); + } + + const decisions: RouteDecision[] = []; + for (let i = 0; i < requests.length; i++) { + const request = requests[i]; + if (request === null || typeof request !== "object" || Array.isArray(request)) { + throw new TypeError( + `request ${i} must be a dict, got ${request === null ? "NoneType" : Array.isArray(request) ? "list" : typeof request}`, + ); + } + if (!("state" in request)) { + throw new Error(`request ${i} is missing required key 'state'`); + } + if (!("questions" in request)) { + throw new Error(`request ${i} is missing required key 'questions'`); + } + const questions = (request as unknown as Record).questions; + if (questions === null || typeof questions !== "object" || Array.isArray(questions)) { + throw new TypeError( + `request ${i} 'questions' must be a dict, got ${questions === null ? "NoneType" : Array.isArray(questions) ? "list" : typeof questions}`, + ); + } + + decisions.push( + this.route( + request.state, + request.questions as Record, + { + model: request.model, + task: request.task, + lang: request.lang, + langGuess: request.langGuess ?? request.lang_guess, + }, + ), + ); + } + + return decisions; + } + + async predictBatch( + requests: RouterBatchRequest[], + opts?: RouterBatchOptions | number | null, + ): Promise { + const decisions = this.routeBatch(requests); + if (decisions.length === 0) { + return []; + } + + const batchSize = typeof opts === "number" ? opts : (opts?.batchSize ?? opts?.batch_size ?? null); + + const groups = new Map(); + for (let i = 0; i < decisions.length; i++) { + const decision = decisions[i] as unknown as Record; + const modelName = decision["model"] as string; + let idxs = groups.get(modelName); + if (!idxs) { + idxs = []; + groups.set(modelName, idxs); + } + idxs.push(i); + } + + const results: Array = new Array(requests.length).fill(null); + const active = composeHooks(this.hooks); + const raiseErrors = this.hooksRaise; + + for (const [modelName, indices] of groups.entries()) { + const agent = (await this.load(modelName)) as { + predictBatch?( + states: unknown[], + questions: Record, + opts?: { batchSize?: number | null; maxLen?: number | null; headMaxLen?: number | null }, + ): Promise; + predict_batch?( + states: unknown[], + questions: Record, + batch_size?: number | null, + overrides?: Record, + ): Promise; + systemOne?( + state: unknown, + questions: Record, + opts?: Record, + ): Promise; + }; + + const started: PredictContext[] = []; + try { + for (const i of indices) { + const req = requests[i]; + const ctx = new PredictContext({ + states: [req.state], + questions: req.questions as Record, + decision: { ...(decisions[i] as unknown as Record) }, + model: modelName, + agent, + router: this, + }); + started.push(ctx); + dispatch(active, "onPredictStart", ctx, { raiseErrors }); + } + + const questionGroups: Array<{ + questions: Record; + schema: string; + maxLen: number | null; + headMaxLen: number | null; + items: Array<[number, PredictContext]>; + }> = []; + + for (let k = 0; k < indices.length; k++) { + const i = indices[k]; + const ctx = started[k]; + + if (ctx.results !== null) { + for (const res of ctx.results) { + if (res && typeof res === "object" && !("routing" in res)) { + (res as Record).routing = { + ...(decisions[i] as unknown as Record), + }; + } + } + continue; + } + + const schema = _questionSchema(ctx.questions); + const maxLen = ctx.maxLen ?? null; + const headMaxLen = ctx.headMaxLen ?? null; + + let found = false; + for (const g of questionGroups) { + if (g.schema === schema && g.maxLen === maxLen && g.headMaxLen === headMaxLen) { + g.items.push([i, ctx]); + found = true; + break; + } + } + if (!found) { + questionGroups.push({ + questions: ctx.questions as Record, + schema, + maxLen, + headMaxLen, + items: [[i, ctx]], + }); + } + } + + for (const group of questionGroups) { + const items = group.items; + const states = items.map(([, ctx]) => ctx.states[0]); + + const predictOpts: { + batchSize?: number | null; + maxLen?: number | null; + headMaxLen?: number | null; + } = {}; + if (batchSize !== null && batchSize !== undefined) { + predictOpts.batchSize = batchSize; + } + if (group.maxLen !== null && group.maxLen !== undefined) { + predictOpts.maxLen = group.maxLen; + } + if (group.headMaxLen !== null && group.headMaxLen !== undefined) { + predictOpts.headMaxLen = group.headMaxLen; + } + + const hasOpts = Object.keys(predictOpts).length > 0; + let batchResults: SystemOneResult[]; + if (typeof agent.predictBatch === "function") { + batchResults = hasOpts + ? await agent.predictBatch(states, group.questions, predictOpts) + : await agent.predictBatch(states, group.questions); + } else if (typeof agent.predict_batch === "function") { + const pyOverrides: Record = {}; + if (group.maxLen !== null) pyOverrides["max_len"] = group.maxLen; + if (group.headMaxLen !== null) pyOverrides["head_max_len"] = group.headMaxLen; + batchResults = await agent.predict_batch( + states, + group.questions, + batchSize, + pyOverrides, + ); + } else if (typeof agent.systemOne === "function") { + batchResults = await Promise.all( + states.map((s) => (hasOpts ? agent.systemOne!(s, group.questions, predictOpts) : agent.systemOne!(s, group.questions))), + ); + } else { + throw new Error(`agent for model "${modelName}" does not implement predictBatch`); + } + + if (!Array.isArray(batchResults) || batchResults.length !== items.length) { + throw new Error( + `internal error: Agent.predict_batch returned ${batchResults?.length ?? 0} results for ${items.length} states`, + ); + } + + for (let m = 0; m < items.length; m++) { + const [i, ctx] = items[m]; + const res = batchResults[m] as RoutedResult; + res.routing = { ...(decisions[i] as unknown as RouteDecision) }; + ctx.results = [res as unknown as Record]; + } + } + } catch (exc) { + for (const ctx of started) { + if (ctx.results === null) { + ctx.error = exc; + try { + dispatch(active, "onError", ctx, { raiseErrors }); + } catch { + // A failing onError hook must not hide the failure that triggered it. + } + } + } + try { + this._endContexts(active, started, raiseErrors); + } catch { + // Do not mask original failure + } + throw exc; + } + + this._endContexts(active, started, raiseErrors); + for (let k = 0; k < indices.length; k++) { + const i = indices[k]; + const ctx = started[k]; + results[i] = (ctx.results as unknown as RoutedResult[])[0]; + } + } + + if (results.some((r) => r === null)) { + throw new Error("internal error: batch execution did not produce every result"); + } + + return results as RoutedResult[]; + } + + private _endContexts( + active: Hook[], + contexts: PredictContext[], + raiseErrors: boolean, + ): void { + const now = + typeof performance !== "undefined" && typeof performance.now === "function" + ? performance.now() + : Date.now(); + for (const ctx of contexts) { + ctx.elapsedMs = now - ctx.startedAt; + if (ctx.results !== null) { + ctx.usage = aggregateUsage(ctx.results); + } + } + let firstError: unknown = null; + for (const ctx of contexts) { + try { + dispatch(active, "onPredictEnd", ctx, { raiseErrors }); + } catch (hookErr) { + if (ctx.error !== null) { + // Hook error ignored on already-failed context so primary error is not masked + } else if (firstError === null) { + firstError = hookErr; + } + } + } + if (firstError !== null) { + throw firstError; + } + } } diff --git a/laya-ts/tests/agent.test.ts b/laya-ts/tests/agent.test.ts index 76c810fdd..325576fc6 100644 --- a/laya-ts/tests/agent.test.ts +++ b/laya-ts/tests/agent.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { Agent } from "../src/agent.js"; +import { Agent, defaultTokenizer, type ChoiceAnswer, type NoulAnswer } from "../src/agent.js"; const fakeProvider = () => ({ async runEncoder(_b: any) { return { lastHidden: [[1, 0], [0, 1]] }; }, async runHead(_h: any) { return { logits: [[2, 0]], act: [[3, 0]] }; }, @@ -21,4 +21,269 @@ describe("agent", () => { const a = new Agent({ provider: fakeProvider() } as any); await expect(a.systemOne("hi", { q: { type: "nope" } as any })).rejects.toThrow('question "q"'); }); + + it("empty batch returns [] without calling provider", async () => { + const log: { encoderBatches: number[][][]; headCalls: number } = { encoderBatches: [], headCalls: 0 }; + const dynamicProvider = { + async runEncoder(b: { inputIds: number[][] }) { + log.encoderBatches.push(b.inputIds); + return { lastHidden: b.inputIds.map((row) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, b: { markerPos: number[][] }) { + log.headCalls++; + return { + logits: b.markerPos.map((m) => m.map((_, i) => i + 1)), + act: b.markerPos.map(() => [3, 0]), + }; + }, + }; + const a = new Agent({ provider: dynamicProvider } as any); + const res = await a.predictBatch([], { + d: { type: "choice", instructions: "q?", criteria: { x: "yes", y: "no" } }, + }); + expect(res).toEqual([]); + expect(log.encoderBatches.length).toBe(0); + expect(log.headCalls).toBe(0); + }); + + it("evaluates multiple states in a single shared forward pass", async () => { + const log: { encoderBatches: number[][][]; headCalls: number } = { encoderBatches: [], headCalls: 0 }; + const dynamicProvider = { + async runEncoder(b: { inputIds: number[][] }) { + log.encoderBatches.push(b.inputIds); + return { lastHidden: b.inputIds.map((row) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, b: { markerPos: number[][] }) { + log.headCalls++; + return { + logits: b.markerPos.map((m) => m.map((_, i) => i + 1)), + act: b.markerPos.map(() => [3, 0]), + }; + }, + }; + const a = new Agent({ provider: dynamicProvider } as any); + const states = ["first state", "second state", "third state"]; + const questions = { + d: { type: "choice", instructions: "q?", criteria: { x: "yes", y: "no" } }, + }; + const res = await a.predictBatch(states, questions); + expect(res.length).toBe(3); + expect(log.encoderBatches.length).toBe(1); // 1 single forward pass for all 3 states + expect(log.encoderBatches[0].length).toBe(3); // 3 rows in inputIds + expect(log.headCalls).toBe(1); + for (const r of res) { + expect((r.answers.d as ChoiceAnswer).choice).toBeDefined(); + expect(r.usage.input_tokens).toBeGreaterThan(0); + } + }); + + it("batch result matches sequential single-state prediction", async () => { + const dynamicProvider = { + async runEncoder(b: { inputIds: number[][] }) { + return { lastHidden: b.inputIds.map((row) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, b: { markerPos: number[][] }) { + return { + logits: b.markerPos.map((m) => m.map((_, i) => i + 1)), + act: b.markerPos.map(() => [3, 0]), + }; + }, + }; + const a = new Agent({ provider: dynamicProvider } as any); + const states = ["ticket about refund", "urgent password reset", "general feedback"]; + const questions = { + topic: { type: "choice", instructions: "topic?", criteria: { billing: "refunds", auth: "login", other: "misc" } }, + urgent: { type: "noul", instructions: "is urgent?" }, + }; + + const sequential = await Promise.all( + states.map((state) => a.predict(state, questions)), + ); + const batched = await a.predictBatch(states, questions); + + expect(batched.length).toBe(sequential.length); + for (let i = 0; i < states.length; i++) { + expect(batched[i].model).toBe(sequential[i].model); + const bTopic = batched[i].answers.topic as ChoiceAnswer; + const sTopic = sequential[i].answers.topic as ChoiceAnswer; + expect(bTopic.choice).toBe(sTopic.choice); + expect(bTopic.probabilities).toEqual(sTopic.probabilities); + expect(bTopic.confidence).toBe(sTopic.confidence); + const bUrgent = batched[i].answers.urgent as NoulAnswer; + const sUrgent = sequential[i].answers.urgent as NoulAnswer; + expect(bUrgent.noul).toBe(sUrgent.noul); + expect(bUrgent.confidence).toBe(sUrgent.confidence); + } + }); + + it("chunks states into multiple forward passes when batchSize is specified", async () => { + const log: { encoderBatches: number[][][]; headCalls: number } = { encoderBatches: [], headCalls: 0 }; + const dynamicProvider = { + async runEncoder(b: { inputIds: number[][] }) { + log.encoderBatches.push(b.inputIds); + return { lastHidden: b.inputIds.map((row) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, b: { markerPos: number[][] }) { + log.headCalls++; + return { + logits: b.markerPos.map((m) => m.map((_, i) => i + 1)), + act: b.markerPos.map(() => [3, 0]), + }; + }, + }; + const a = new Agent({ provider: dynamicProvider } as any); + const states = ["s1", "s2", "s3", "s4", "s5"]; + const questions = { + d: { type: "choice", instructions: "q?", criteria: { x: "yes", y: "no" } }, + }; + const res = await a.predictBatch(states, questions, { batchSize: 2 }); + expect(res.length).toBe(5); + // 5 states with batchSize 2 -> chunks of 2, 2, 1 + expect(log.encoderBatches.length).toBe(3); + expect(log.encoderBatches[0].length).toBe(2); + expect(log.encoderBatches[1].length).toBe(2); + expect(log.encoderBatches[2].length).toBe(1); + expect(log.headCalls).toBe(3); + }); + + it("rejects non-array states in predictBatch", async () => { + const a = new Agent({ provider: fakeProvider() } as any); + await expect(a.predictBatch("single string" as any, {})).rejects.toThrow(TypeError); + }); + + it("rejects non-positive batchSize", async () => { + const a = new Agent({ provider: fakeProvider() } as any); + await expect(a.predictBatch(["s1"], {}, { batchSize: 0 })).rejects.toThrow("batchSize must be a positive integer"); + await expect(a.predictBatch(["s1"], {}, { batchSize: -2 })).rejects.toThrow("batchSize must be a positive integer"); + }); + + it("fires hooks in batch mode and records aggregated usage", async () => { + const events: string[] = []; + let endUsage: any = null; + const dynamicProvider = { + async runEncoder(b: { inputIds: number[][] }) { + return { lastHidden: b.inputIds.map((row) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, b: { markerPos: number[][] }) { + return { + logits: b.markerPos.map((m) => m.map((_, i) => i + 1)), + act: b.markerPos.map(() => [3, 0]), + }; + }, + }; + const a = new Agent({ + provider: dynamicProvider, + hooks: [ + { + onPredictStart(_ctx: any) { events.push("start"); }, + onPredictEnd(ctx: any) { + events.push("end"); + endUsage = ctx.usage; + }, + }, + ], + } as any); + + const res = await a.predictBatch(["s1", "s2"], { + q: { type: "noul", instructions: "?" }, + }); + expect(events).toEqual(["start", "end"]); + expect(res.length).toBe(2); + expect(endUsage.input_tokens).toBe(res[0].usage.input_tokens + res[1].usage.input_tokens); + }); + + it("ctx.skip() short-circuits batch prediction without calling provider", async () => { + let providerCalls = 0; + const provider = { + async runEncoder() { providerCalls++; return { lastHidden: [] }; }, + async runHead() { providerCalls++; return { logits: [], act: [] }; }, + }; + const cached: any = [ + { model: "cached-model", answers: { q: { type: "noul", noul: 1, confidence: 1 } }, usage: { input_tokens: 10, output_tokens: 0 } }, + { model: "cached-model", answers: { q: { type: "noul", noul: 0, confidence: 1 } }, usage: { input_tokens: 12, output_tokens: 0 } }, + ]; + let endCtx: any = null; + const a = new Agent({ + provider: provider as any, + hooks: [ + { + onPredictStart(ctx: any) { ctx.skip(cached); }, + onPredictEnd(ctx: any) { endCtx = ctx; }, + }, + ], + } as any); + + const res = await a.predictBatch(["s1", "s2"], { q: { type: "noul", instructions: "?" } }); + expect(res).toBe(cached); + expect(providerCalls).toBe(0); + expect(endCtx.results).toBe(cached); + expect(endCtx.usage).toEqual({ input_tokens: 22, output_tokens: 0 }); + }); + + it("evaluates mutated states when onPredictStart rewrites ctx.states", async () => { + const encodedTexts: string[] = []; + const tok = { + ...defaultTokenizer(), + encode(text: string): number[] { + encodedTexts.push(text); + return [1, 2, 3]; + }, + }; + const dynamicProvider = { + async runEncoder(b: { inputIds: number[][] }) { + return { lastHidden: b.inputIds.map((row) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, b: { markerPos: number[][] }) { + return { + logits: b.markerPos.map((m) => m.map((_, i) => i + 1)), + act: b.markerPos.map(() => [3, 0]), + }; + }, + }; + const a = new Agent({ + provider: dynamicProvider as any, + tok, + hooks: [ + { + onPredictStart(ctx: any) { + ctx.states = ["rewritten-a", "rewritten-b"]; + }, + }, + ], + } as any); + + const res = await a.predictBatch(["orig-1", "orig-2"], { + q: { type: "noul", instructions: "?" }, + }); + expect(res.length).toBe(2); + expect(encodedTexts).toContain("rewritten-a"); + expect(encodedTexts).toContain("rewritten-b"); + expect(encodedTexts).not.toContain("orig-1"); + expect(encodedTexts).not.toContain("orig-2"); + }); + + it("honors maxLen override in predictBatch", async () => { + let capturedInputIds: number[][] = []; + const dynamicProvider = { + async runEncoder(b: { inputIds: number[][] }) { + capturedInputIds = b.inputIds; + return { lastHidden: b.inputIds.map((row) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, b: { markerPos: number[][] }) { + return { + logits: b.markerPos.map((m) => m.map((_, i) => i + 1)), + act: b.markerPos.map(() => [3, 0]), + }; + }, + }; + const a = new Agent({ provider: dynamicProvider as any } as any); + const longState = "very long state text ".repeat(30); + await a.predictBatch([longState, longState], { + q: { type: "noul", instructions: "?" }, + }, { maxLen: 30 }); + + expect(capturedInputIds.length).toBe(2); + expect(capturedInputIds[0].length).toBeLessThanOrEqual(30); + expect(capturedInputIds[1].length).toBeLessThanOrEqual(30); + }); }); diff --git a/laya-ts/tests/router.test.ts b/laya-ts/tests/router.test.ts index 929c08edc..50880c513 100644 --- a/laya-ts/tests/router.test.ts +++ b/laya-ts/tests/router.test.ts @@ -1,5 +1,70 @@ import { describe, expect, it } from "vitest"; -import { Router, normaliseName } from "../src/router.js"; +import { Router, normaliseName, _questionSchema, type RouterBatchRequest } from "../src/router.js"; +import type { QuestionDef, SystemOneResult } from "../src/agent.js"; +import type { PredictContext } from "../src/hooks.js"; + +const Q: Record = { + intent: { type: "noul", instructions: "Relevant?" }, +}; + +function req(state: unknown, overrides: Partial = {}): RouterBatchRequest { + return { state, questions: Q, ...overrides }; +} + +class BatchFakeAgent { + checkpoint: string; + calls: Array<{ states: unknown[]; questions: Record; opts?: Record }> = []; + + constructor(checkpoint: string) { + this.checkpoint = checkpoint; + } + + async predictBatch( + states: unknown[], + questions: Record, + opts?: Record, + ): Promise { + this.calls.push({ states: [...states], questions, opts: opts ? { ...opts } : undefined }); + if (states.includes("boom")) { + throw new Error("inference failed"); + } + return states.map((s) => ({ + model: "fake", + answers: { seen: s } as any, + usage: { input_tokens: typeof s === "string" ? s.length : 1, output_tokens: 0 }, + })); + } + + async systemOne( + state: unknown, + questions: Record, + opts?: Record, + ): Promise { + const res = await this.predictBatch([state], questions, opts); + return res[0]; + } +} + +function makeRouterWithLoader(capacity = 2, autoTaskDetection = false) { + const agents: Record = { + english: new BatchFakeAgent("english"), + multilingual: new BatchFakeAgent("multilingual"), + "typed-decisions": new BatchFakeAgent("typed-decisions"), + }; + const built: string[] = []; + + const router = new Router({ + maxLoaded: capacity, + autoTaskDetection, + loader: async (name) => { + built.push(name); + return agents[name]; + }, + }); + + return { router, agents, built }; +} + describe("router", () => { it("normalises aliases", () => { expect(normaliseName("en")).toBe("english"); }); it("rejects unknown model", () => { expect(() => normaliseName("jev-1")).toThrow(); }); @@ -16,3 +81,351 @@ describe("router", () => { expect(r.route("text", {}, { langGuess: () => "ro" }).model).toBe("multilingual"); }); }); + +describe("router batching validation and routing", () => { + it("rejects invalid requests argument", async () => { + const { router } = makeRouterWithLoader(); + expect(() => router.routeBatch(null as any)).toThrow(TypeError); + expect(() => router.routeBatch({} as any)).toThrow("requests must be a sequence"); + expect(() => router.routeBatch("text" as any)).toThrow("requests must be a sequence"); + await expect(router.predictBatch(null as any)).rejects.toThrow("requests must be a sequence"); + }); + + it("rejects malformed items before loading", async () => { + const { router, built } = makeRouterWithLoader(); + expect(() => router.routeBatch([null as any])).toThrow("request 0 must be a dict"); + expect(() => router.routeBatch([{ questions: Q } as any])).toThrow("request 0 is missing required key 'state'"); + expect(() => router.routeBatch([{ state: "x" } as any])).toThrow("request 0 is missing required key 'questions'"); + expect(() => router.routeBatch([req("x"), { state: "y", questions: null } as any])).toThrow("request 1 'questions' must be a dict"); + expect(() => router.routeBatch([req("x"), req("y", { model: "invalid" })])).toThrow("unknown model"); + + await expect(router.predictBatch([null as any])).rejects.toThrow("request 0 must be a dict"); + expect(built).toEqual([]); + expect(router.loaded).toEqual([]); + }); + + it("handles empty batch", async () => { + const { router, built } = makeRouterWithLoader(); + expect(router.routeBatch([])).toEqual([]); + expect(await router.predictBatch([])).toEqual([]); + expect(built).toEqual([]); + expect(router.loaded).toEqual([]); + }); + + it("routeBatch forwards langGuess without loading checkpoints", () => { + const { router, built } = makeRouterWithLoader(); + const decisions = router.routeBatch([ + req("hola", { langGuess: "es" }), + req("hello", { lang_guess: "en-US" }), + ]); + expect(decisions.map((d) => d.model)).toEqual(["multilingual", "english"]); + expect(built).toEqual([]); + expect(router.loaded).toEqual([]); + }); +}); + +describe("router predictBatch execution, ordering, and LRU", () => { + it.each([ + { capacity: 1, expectedBuilt: ["english", "multilingual", "typed-decisions"], expectedLoaded: ["typed-decisions"] }, + { capacity: 2, expectedBuilt: ["english", "multilingual", "typed-decisions"], expectedLoaded: ["multilingual", "typed-decisions"] }, + { capacity: 3, expectedBuilt: ["english", "multilingual", "typed-decisions"], expectedLoaded: ["english", "multilingual", "typed-decisions"] }, + ])("mixed groups keep original order and respect LRU (capacity=$capacity)", async ({ capacity, expectedBuilt, expectedLoaded }) => { + const { router, agents, built } = makeRouterWithLoader(capacity, true); + const typedQuestions = { + action: { type: "noul", instructions: "?" }, + needs_review: { type: "noul", instructions: "?" }, + outcome: { type: "noul", instructions: "?" }, + risk: { type: "noul", instructions: "?" }, + urgency: { type: "noul", instructions: "?" }, + }; + + const items: RouterBatchRequest[] = [ + req("English text one"), + req("مرحبا"), + req("English text two"), + { state: "decision", questions: typedQuestions }, + req("forced", { model: "ml" }), + req("forced english", { lang: "en" }), + req("explicit task", { task: "typed_decisions" }), + ]; + + const decisions = router.routeBatch(items); + expect(router.loaded).toEqual([]); + + const results = await router.predictBatch(items); + expect(results.length).toBe(items.length); + expect(results.map((r) => (r.answers as any).seen)).toEqual(items.map((i) => i.state)); + expect(results.map((r) => r.routing)).toEqual(decisions); + expect(built).toEqual(expectedBuilt); + expect(router.loaded).toEqual(expectedLoaded); + + expect(agents.english.calls.map((c) => c.states)).toEqual([ + ["English text one", "English text two", "forced english"], + ]); + expect(agents.multilingual.calls.map((c) => c.states)).toEqual([ + ["مرحبا", "forced"], + ]); + expect(agents["typed-decisions"].calls.map((c) => c.states)).toEqual([ + ["decision"], + ["explicit task"], + ]); + }); + + it("same checkpoint and same questions coalesce into one agent call with batchSize", async () => { + const { router, agents } = makeRouterWithLoader(2); + const items = [ + req("one", { model: "english" }), + req("two", { model: "english" }), + req("three", { model: "english" }), + ]; + + const results = await router.predictBatch(items, { batchSize: 2 }); + expect(results.map((r) => (r.answers as any).seen)).toEqual(["one", "two", "three"]); + expect(agents.english.calls.length).toBe(1); + expect(agents.english.calls[0].states).toEqual(["one", "two", "three"]); + expect(agents.english.calls[0].opts).toEqual({ batchSize: 2 }); + }); + + it("forwards numeric batchSize parameter directly to agent.predictBatch", async () => { + const { router, agents } = makeRouterWithLoader(2); + const items = [ + req("one", { model: "english" }), + req("two", { model: "english" }), + req("three", { model: "english" }), + ]; + + const results = await router.predictBatch(items, 2); + expect(results.map((r) => (r.answers as any).seen)).toEqual(["one", "two", "three"]); + expect(agents.english.calls.length).toBe(1); + expect(agents.english.calls[0].states).toEqual(["one", "two", "three"]); + expect(agents.english.calls[0].opts).toEqual({ batchSize: 2 }); + }); + + it("same checkpoint and different questions split agent batches", async () => { + const { router, agents } = makeRouterWithLoader(2); + const q2 = { risk: { type: "noul", instructions: "Risky?" } }; + const items = [ + req("one", { model: "english" }), + req("two", { model: "english", questions: q2 }), + req("three", { model: "english" }), + ]; + + const results = await router.predictBatch(items); + expect(results.map((r) => (r.answers as any).seen)).toEqual(["one", "two", "three"]); + expect(agents.english.calls.length).toBe(2); + expect(agents.english.calls[0].states).toEqual(["one", "three"]); + expect(agents.english.calls[0].questions).toBe(Q); + expect(agents.english.calls[1].states).toEqual(["two"]); + expect(agents.english.calls[1].questions).toBe(q2); + }); + + it("equal questions with different option order score separately (#166)", async () => { + const { router, agents } = makeRouterWithLoader(1); + const ordered = { + intent: { type: "choice", instructions: "Pick", criteria: { zulu: "last", alpha: "first" } }, + }; + const reordered = { + intent: { type: "choice", instructions: "Pick", criteria: { alpha: "first", zulu: "last" } }, + }; + + expect(_questionSchema(ordered)).not.toBe(_questionSchema(reordered)); + + const items: RouterBatchRequest[] = [ + { state: "one", questions: ordered, model: "english" }, + { state: "two", questions: reordered, model: "english" }, + ]; + + const results = await router.predictBatch(items); + expect(results.length).toBe(2); + expect(agents.english.calls.length).toBe(2); + expect(Object.keys(agents.english.calls[0].questions.intent.criteria as any)).toEqual(["zulu", "alpha"]); + expect(Object.keys(agents.english.calls[1].questions.intent.criteria as any)).toEqual(["alpha", "zulu"]); + }); +}); + +describe("router batch hooks and error handling", () => { + it("partial ctx.skip() bypasses agent, preserves routing, and other requests infer normally", async () => { + const cachedX = { + model: "cached-model", + answers: { seen: "cached-x" } as any, + usage: { input_tokens: 0, output_tokens: 0 }, + }; + const { router, agents } = makeRouterWithLoader(); + router.addHook({ + onPredictStart(ctx: PredictContext) { + if (ctx.states[0] === "x") { + ctx.skip([cachedX]); + } + }, + }); + + const results = await router.predictBatch([ + req("x", { model: "english" }), + req("y", { model: "english" }), + ]); + + expect(agents.english.calls.length).toBe(1); + expect(agents.english.calls[0].states).toEqual(["y"]); + + expect((results[0].answers as any).seen).toBe("cached-x"); + expect(results[0].routing.model).toBe("english"); + expect((results[1].answers as any).seen).toBe("y"); + expect(results[1].routing.model).toBe("english"); + }); + + it("onPredictStart state rewrite reaches the agent", async () => { + const { router, agents } = makeRouterWithLoader(); + router.addHook({ + onPredictStart(ctx: PredictContext) { + ctx.states = [String(ctx.states[0]).replace("secret", "[redacted]")]; + }, + }); + + const out = await router.predictBatch([ + req("a secret", { model: "english" }), + req("b secret", { model: "multilingual" }), + req("c secret", { model: "english" }), + ]); + + expect(agents.english.calls[0].states).toEqual(["a [redacted]", "c [redacted]"]); + expect(agents.multilingual.calls[0].states).toEqual(["b [redacted]"]); + expect(out.map((o) => (o.answers as any).seen)).toEqual(["a [redacted]", "b [redacted]", "c [redacted]"]); + }); + + it("onPredictStart question mutation creates a separate question group", async () => { + const { router, agents } = makeRouterWithLoader(); + const qOther = { other: { type: "noul", instructions: "Other?" } }; + router.addHook({ + onPredictStart(ctx: PredictContext) { + if (ctx.states[0] === "b") { + ctx.questions = qOther; + } + }, + }); + + await router.predictBatch([ + req("a", { model: "english" }), + req("b", { model: "english" }), + req("c", { model: "english" }), + ]); + + expect(agents.english.calls.length).toBe(2); + expect(agents.english.calls[0].states).toEqual(["a", "c"]); + expect(agents.english.calls[0].questions).toBe(Q); + expect(agents.english.calls[1].states).toEqual(["b"]); + expect(agents.english.calls[1].questions).toBe(qOther); + }); + + it("token budget override creates a separate question group and passes opts", async () => { + const { router, agents } = makeRouterWithLoader(); + router.addHook({ + onPredictStart(ctx: PredictContext) { + if (ctx.states[0] === "long") { + ctx.maxLen = 1024; + } + }, + }); + + await router.predictBatch([ + req("short", { model: "english" }), + req("long", { model: "english" }), + req("short again", { model: "english" }), + ]); + + expect(agents.english.calls.length).toBe(2); + expect(agents.english.calls[0].states).toEqual(["short", "short again"]); + expect(agents.english.calls[0].opts).toBeUndefined(); + expect(agents.english.calls[1].states).toEqual(["long"]); + expect(agents.english.calls[1].opts).toEqual({ maxLen: 1024 }); + }); + + it("failure dispatches onError only to unskipped requests, and onPredictEnd to all started requests", async () => { + const events: Array<{ type: string; state: unknown }> = []; + let cachedEndedHasError: unknown = null; + let cachedEndedHasResults: unknown = null; + + const cached = { + model: "cached", + answers: { seen: "cached" } as any, + usage: { input_tokens: 0, output_tokens: 0 }, + }; + + const { router } = makeRouterWithLoader(); + router.addHook({ + onPredictStart(ctx: PredictContext) { + events.push({ type: "start", state: ctx.states[0] }); + if (ctx.states[0] === "cached") { + ctx.skip([cached]); + } + }, + onError(ctx: PredictContext) { + events.push({ type: "error", state: ctx.states[0] }); + }, + onPredictEnd(ctx: PredictContext) { + events.push({ type: "end", state: ctx.states[0] }); + if (ctx.states[0] === "cached") { + cachedEndedHasError = ctx.error; + cachedEndedHasResults = ctx.results; + } + }, + }); + + await expect( + router.predictBatch([ + req("ok", { model: "multilingual" }), + req("cached", { model: "english" }), + req("boom", { model: "english" }), + ]), + ).rejects.toThrow("inference failed"); + + // The first group (multilingual with "ok") settled cleanly + expect(events.filter((e) => e.state === "ok").map((e) => e.type)).toEqual(["start", "end"]); + + // In the failing group: + // "cached" started, was skipped, was NOT errored, and ended cleanly + expect(events.filter((e) => e.state === "cached").map((e) => e.type)).toEqual(["start", "end"]); + expect(cachedEndedHasError).toBeNull(); + expect(cachedEndedHasResults).toBeDefined(); + + // "boom" started, errored, and ended + expect(events.filter((e) => e.state === "boom").map((e) => e.type)).toEqual(["start", "error", "end"]); + }); + + it("elapsedMs and usage are set for all contexts before any onPredictEnd runs", async () => { + const groupCtxs: PredictContext[] = []; + const elapsedWhenFirstEndRan: boolean[][] = []; + + const { router } = makeRouterWithLoader(); + router.addHook({ + onPredictStart(ctx: PredictContext) { + groupCtxs.push(ctx); + }, + onPredictEnd() { + if (elapsedWhenFirstEndRan.length === 0) { + elapsedWhenFirstEndRan.push(groupCtxs.map((c) => c.elapsedMs !== null)); + } + }, + }); + + await router.predictBatch([ + req("a", { model: "english" }), + req("b", { model: "english" }), + req("c", { model: "english" }), + ]); + + expect(elapsedWhenFirstEndRan).toEqual([[true, true, true]]); + }); + + it("onRoute hook may replace decision with a plain dict", async () => { + const { router, agents } = makeRouterWithLoader(); + router.addHook({ + onRoute(ctx: PredictContext) { + ctx.decision = { ...ctx.decision, model: "multilingual" }; + }, + }); + + const out = await router.predictBatch([req("pinned", { model: "english" })]); + expect(agents.multilingual.calls[0].states).toEqual(["pinned"]); + expect(out[0].routing.model).toBe("multilingual"); + }); +});