diff --git a/laya-ts/README.md b/laya-ts/README.md index 4b124855e..ff94451a2 100644 --- a/laya-ts/README.md +++ b/laya-ts/README.md @@ -82,9 +82,19 @@ const out = await predictShortlist(agent, state, questions, embedFn, 20); ```bash node laya-ts/examples/try-ml.mjs # needs ./model-ml from the export step +node laya-ts/examples/try-batch.mjs # bulk predictMany over 3 states, one forward pass per model node laya-ts/examples/snake.mjs --ticks 50 # autonomous snake demo, headless smoke (live TUI without --ticks) ``` +## Batch (one forward pass for N states) + +```ts +const out = await agent.predictMany(states, questions); // SystemOneResult[] +const routed = await router.predictMany(states, questions); // + routing per state +import { q, best, topK } from "laya-ts"; +const qq = { d: q.choice("pick", { refund: "money back", other: "rest" }) }; +``` + ## Packaging ponytail: CJS/browser-field dual build + tsconfig tests-include deferred — Task 7 verified ESM-only; CJS needs second tsc config + export-map change, untested. Add when a CJS consumer or browser-field swap is requested. diff --git a/laya-ts/examples/snake.mjs b/laya-ts/examples/snake.mjs index dd901de0c..17afe10ee 100644 --- a/laya-ts/examples/snake.mjs +++ b/laya-ts/examples/snake.mjs @@ -1,4 +1,5 @@ import { Agent } from "../dist/index.js"; +import { existsSync } from "node:fs"; // ---- CLI (no deps) ---- const args = process.argv.slice(2); @@ -6,7 +7,7 @@ const opt = (name, def) => { const i = args.indexOf(name); return i === -1 || i + 1 >= args.length ? def : args[i + 1]; }; -const MODEL_DIR = opt("--model", "./model-ml"); +const MODEL_DIR = opt("--model", ["./model-ml", "laya-ts/examples/model-ml", "./examples/model-ml"].find((d) => existsSync(d)) ?? "./model-ml"); const TICK_MS = Math.max(20, Number(opt("--tick", "120")) || 120); const W = Math.max(10, Number(opt("--width", "20")) || 20); const H = Math.max(6, Number(opt("--height", "12")) || 12); diff --git a/laya-ts/examples/try-batch.mjs b/laya-ts/examples/try-batch.mjs new file mode 100644 index 000000000..4ace33af6 --- /dev/null +++ b/laya-ts/examples/try-batch.mjs @@ -0,0 +1,43 @@ +import { Agent, Router } from "../dist/index.js"; +import { existsSync } from "node:fs"; + +// run as: node laya-ts/examples/try-batch.mjs (repo root) or node examples/try-batch.mjs (laya-ts dir) +// --model overrides; otherwise first existing dir wins (repo ships weights at laya-ts/examples/model-ml) +const args = process.argv.slice(2); +const flag = (name) => { + const i = args.indexOf(name); + return i === -1 || i + 1 >= args.length ? null : args[i + 1]; +}; +const MODEL_DIR = flag("--model") ?? + ["./model-ml", "laya-ts/examples/model-ml", "./examples/model-ml"].find((d) => existsSync(d)) ?? "./model-ml"; +const agent = await Agent.load(MODEL_DIR); + +const states = [ + { body: "charged twice, refund please" }, + { body: "मुझसे दो बार शुल्क लिया गया" }, + { body: "Der Kunde wurde zweimal belastet" }, +]; +const questions = { + department: { + type: "choice", + instructions: "Which department should handle this request?", + criteria: { billing: "invoices, payments, refunds", technical: "bugs, outages", other: "rest" }, + }, +}; + +// One forward pass for all states (vs N sequential predicts). +const batched = await agent.predictMany(states, questions); +for (const [i, r] of batched.entries()) { + const a = r.answers.department; + console.log(`batched[${i}]:`, a?.choice, `conf=${a?.confidence}`, JSON.stringify(a?.probabilities)); +} + +// Router groups by language, one load per checkpoint, order preserved. +// Same local weights answer both groups so the demo runs offline. +const router = new Router(); +router.attach("english", agent); +router.attach("multilingual", agent); +const routed = await router.predictMany(states, questions); +for (const [i, r] of routed.entries()) { + console.log(`routed[${i}]:`, r.routing.model, "->", r.answers.department?.choice); +} diff --git a/laya-ts/examples/try-ml.mjs b/laya-ts/examples/try-ml.mjs index cc4ee6f77..fd600e74a 100644 --- a/laya-ts/examples/try-ml.mjs +++ b/laya-ts/examples/try-ml.mjs @@ -1,6 +1,16 @@ import { Agent, Router } from "../dist/index.js"; +import { existsSync } from "node:fs"; -const agent = await Agent.load("./model-ml"); // run as: node laya-ts/examples/try-ml.mjs (repo root) +// run as: node laya-ts/examples/try-ml.mjs (repo root) or node examples/try-ml.mjs (laya-ts dir) +// --model overrides; otherwise first existing dir wins (repo ships weights at laya-ts/examples/model-ml) +const args = process.argv.slice(2); +const flag = (name) => { + const i = args.indexOf(name); + return i === -1 || i + 1 >= args.length ? null : args[i + 1]; +}; +const MODEL_DIR = flag("--model") ?? + ["./model-ml", "laya-ts/examples/model-ml", "./examples/model-ml"].find((d) => existsSync(d)) ?? "./model-ml"; +const agent = await Agent.load(MODEL_DIR); console.log("loaded:", agent.cfg.encoder ?? "multilingual"); // Direct predict (Hindi -> multilingual weights) diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts index 294b7ad7e..9f1937285 100644 --- a/laya-ts/src/agent.ts +++ b/laya-ts/src/agent.ts @@ -300,6 +300,28 @@ export class Agent extends HookRegistry { return (await this._predictHooked([state], questions, opts))[0]; } + /** + * Evaluate typed questions across many states in a single forward pass. + * + * Same questions for every state (bulk triage). One encoder+head run for + * all S*Q rows instead of S sequential runs. Hooks see all states at once. + */ + async predictMany( + states: unknown[], + questions: Record, + opts: PredictOptions = {}, + ): Promise { + return this._predictHooked([...(states ?? [])], questions, opts); + } + + async systemMany( + states: unknown[], + questions: Record, + opts: PredictOptions = {}, + ): Promise { + return this.predictMany(states, questions, opts); + } + private async _predictHooked( states: unknown[], questions: Record, @@ -315,10 +337,7 @@ export class Agent extends HookRegistry { try { dispatch(active, "onPredictStart", ctx, { raiseErrors }); if (ctx.results === null) { - const out: SystemOneResult[] = []; - for (const st of ctx.states) { - out.push(await this._systemOneCore(st, ctx.questions as Record)); - } + const out = await this._systemManyCore(ctx.states, ctx.questions as Record); ctx.results = out as unknown as Record[]; ctx.model ??= out[0]?.model ?? null; } @@ -347,90 +366,113 @@ export class Agent extends HookRegistry { state: unknown, questions: Record, ): Promise { + return (await this._systemManyCore([state], questions))[0]; + } + + // One encoder+head run for all S*Q rows; chunk upstream if GPU OOM (providers fall back to CPU). + private async _systemManyCore( + states: unknown[], + questions: Record, + ): Promise { const ids = Object.keys(questions ?? {}); + if ((states ?? []).length === 0) return []; if (ids.length === 0) { - return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } }; + return states.map(() => ({ model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } })); } - const items: { ids: number[]; markers: number[]; qtype: number }[] = []; - const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }[] = []; - // The state text is shared by every question and is usually the longest text in the - // sequence — encode it once and compose the per-question prefixes onto it. - const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" ")); - for (const qid of ids) { + const internals = ids.map((qid) => { checkQuestion(qid, questions[qid]); - const q = toInternal(questions[qid]); - internals.push(q); - const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen); - const { ids: seq, markers } = sequenceWithState( - prefix, stAll, this.tok.sepId, this.maxLen, Array.isArray(state), - ); - if (markers.length !== renderOptions(q).length) { - throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`); + return toInternal(questions[qid]); + }); + const optCounts = internals.map((q) => renderOptions(q).length); + const prefixes = internals.map((q) => buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen)); + const items: { ids: number[]; markers: number[]; qtype: number }[] = []; + const stateOfRow: number[] = []; + for (let si = 0; si < states.length; si++) { + const state = states[si]; + const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" ")); + for (let qi = 0; qi < ids.length; qi++) { + const { ids: seq, markers } = sequenceWithState( + prefixes[qi], stAll, this.tok.sepId, this.maxLen, Array.isArray(state), + ); + if (markers.length !== optCounts[qi]) { + throw new Error(`question ${qidStr(ids[qi])} options exceed head_max_len=${this.headMaxLen}`); + } + items.push({ ids: seq, markers, qtype: QTYPES[internals[qi].t] }); + stateOfRow.push(si); } - items.push({ ids: seq, markers, qtype: QTYPES[q.t] }); } const collated = collateItems([items], this.tok.padId); if (!collated) throw new Error("no items to collate"); const batch: Batch = collated; - const nTokens = batch.attentionMask.flat().reduce((a, b) => a + b, 0); const { lastHidden } = await this.provider.runEncoder(batch); const { logits, act } = await this.provider.runHead(lastHidden, batch); - if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < ids.length || act.length < ids.length) { + if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < items.length || act.length < items.length) { throw new Error("model provider returned fewer output rows than input items"); } - for (let r = 0; r < ids.length; r++) { + for (let r = 0; r < items.length; r++) { const k = items[r].markers.length; if (!Array.isArray(logits[r]) || logits[r].length < k || !Array.isArray(act[r]) || act[r].length === 0 || logits[r].some((v) => !Number.isFinite(v)) || act[r].some((v) => !Number.isFinite(v))) { - throw new Error(`model provider returned invalid output for question ${ids[r]}`); + throw new Error(`model provider returned invalid output for question ${ids[r % ids.length]}`); } } - - const answers: Record = {}; - for (let r = 0; r < ids.length; r++) { - const qid = ids[r]; - const q = internals[r]; - const k = items[r].markers.length; - const qt = QTYPES[q.t]; - const bucket = tempBucket(qt, k); - const scale = this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0; - const z = (logits[r] as number[]).slice(0, k).map((v) => v / scale); - const p = softmax(z); - const actRow = (act[r] as number[]) ?? [1, 0]; - const actP = softmax(actRow.slice(0, Math.max(2, actRow.length))); - const ext = { act_probability: r4(actP[0]) }; - if (q.t === "choice") { - const keys = Object.keys(q.crit as Record); - let best = 0; - for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i; - answers[qid] = { - type: "choice", - choice: keys[best], - probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])), - confidence: r4(confidenceFromProbs(p)), - action: ext, - }; - } else if (q.t === "score") { - const exp = p.reduce((a, v, i) => a + i * v, 0); - answers[qid] = { - type: "score", - score: r4(exp), - legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])), - probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])), - confidence: r4(confidenceFromProbs(p)), - action: ext, - }; - } else { - const pt = p[1] ?? 0; - answers[qid] = { - type: "noul", - noul: r4(pt), - confidence: r4(Math.max(pt, 1 - pt)), - action: ext, - }; + const mask = batch.attentionMask; + const tokensPerState = states.map((_, si) => { + let t = 0; + for (let qi = 0; qi < ids.length; qi++) { + const row = mask[si * ids.length + qi] ?? []; + for (const v of row) t += v; } - } - return { model: "laya-rl-agent", answers, usage: { input_tokens: nTokens, output_tokens: 0 } }; + return t; + }); + return states.map((_, si) => { + const answers: Record = {}; + for (let qi = 0; qi < ids.length; qi++) { + const r = si * ids.length + qi; + const qid = ids[qi]; + const q = internals[qi]; + const k = items[r].markers.length; + const qt = QTYPES[q.t]; + const bucket = tempBucket(qt, k); + const scale = this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0; + const z = (logits[r] as number[]).slice(0, k).map((v) => v / scale); + const p = softmax(z); + const actRow = (act[r] as number[]) ?? [1, 0]; + const actP = softmax(actRow.slice(0, Math.max(2, actRow.length))); + const ext = { act_probability: r4(actP[0]) }; + if (q.t === "choice") { + const keys = Object.keys(q.crit as Record); + let best = 0; + for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i; + answers[qid] = { + type: "choice", + choice: keys[best], + probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])), + confidence: r4(confidenceFromProbs(p)), + action: ext, + }; + } else if (q.t === "score") { + const exp = p.reduce((a, v, i) => a + i * v, 0); + answers[qid] = { + type: "score", + score: r4(exp), + legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])), + probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])), + confidence: r4(confidenceFromProbs(p)), + action: ext, + }; + } else { + const pt = p[1] ?? 0; + answers[qid] = { + type: "noul", + noul: r4(pt), + confidence: r4(Math.max(pt, 1 - pt)), + action: ext, + }; + } + } + return { model: "laya-rl-agent", answers, usage: { input_tokens: tokensPerState[si], output_tokens: 0 } }; + }); } async predict( @@ -449,6 +491,8 @@ export class Agent extends HookRegistry { localDir?: string; token?: string | null; numThreads?: number; + signal?: AbortSignal | null; + onProgress?: ((done: number, total: number, file: string) => void) | null; }, ): Promise { const sub = opts?.subfolder ?? null; @@ -460,7 +504,7 @@ export class Agent extends HookRegistry { let provider: SessionProvider; if (isBrowser) { const { loadWebBundle, createWebProvider } = await import("./providers.js"); - const bundle = await loadWebBundle(modelDirOrRepo, { subfolder: sub }); + const bundle = await loadWebBundle(modelDirOrRepo, { subfolder: sub, signal: opts?.signal, onProgress: opts?.onProgress }); cfg = bundle.cfg; tokenizerJson = bundle.tokenizerJson; dir = bundle.dir; @@ -471,6 +515,8 @@ export class Agent extends HookRegistry { subfolder: sub, localDir: opts?.localDir, token: opts?.token, + signal: opts?.signal, + onProgress: opts?.onProgress, }); cfg = bundle.cfg; tokenizerJson = bundle.tokenizerJson; diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts index f46f1b510..e11033629 100644 --- a/laya-ts/src/common.ts +++ b/laya-ts/src/common.ts @@ -58,8 +58,25 @@ export interface QuestionPrefix { } /** The question half of `buildSequence`: everything before the state tokens. Hoisted out so * callers asking several questions about the same state can encode the state text only once. */ +// Per-tokenizer prefix cache (cap 1k, clear on overflow); repeat triage skips re-encode. +const prefixCache = new WeakMap>(); export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ, maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix { + let per = prefixCache.get(tok as object); + if (!per) { + per = new Map(); + prefixCache.set(tok as object, per); + } + const key = JSON.stringify([q.t, q.ins, q.crit, q.labels ?? null, maxLen, headMaxLen, optionOrder ?? null]); + const hit = per.get(key); + if (hit) return hit; + const built = buildQuestionPrefixUncached(tok, q, maxLen, headMaxLen, optionOrder); + if (per.size > 1000) per.clear(); + per.set(key, built); + return built; +} +function buildQuestionPrefixUncached(tok: TokenizerLike, q: InternalQ, + maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix { const maskTok = tok.maskToken; const opts = renderOptions(q); const order = optionOrder ?? opts.map((_, i) => i); @@ -118,6 +135,12 @@ export function tempBucket(qtype: number, k: number): string { const size = k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+"; return `${["choice", "score", "noul"][qtype]}:${size}`; } +/** Max of a length list without spread (Math.max(...arr) throws RangeError past ~100k args). */ +export function maxOf(values: ArrayLike, fallback = 0): number { + let m = fallback; + for (let i = 0; i < values.length; i++) if (values[i] > m) m = values[i]; + return m; +} export interface CollateItem { ids: number[]; markers: number[]; @@ -140,8 +163,12 @@ export interface CollatedBatch { export function collateItems(batch: CollateItem[][], padId: number): CollatedBatch | null { const items = (batch ?? []).flat(); if (items.length === 0) return null; - const L = Math.max(...items.map((it) => it.ids.length)); - const K = Math.max(...items.map((it) => it.markers.length)); + let L = 0; + let K = 0; + for (const it of items) { + if (it.ids.length > L) L = it.ids.length; + if (it.markers.length > K) K = it.markers.length; + } const hasTarget = items.some((it) => "target" in it); const inputIds = items.map((it) => [...it.ids, ...Array(L - it.ids.length).fill(padId)]); const attentionMask = items.map((it) => [...Array(it.ids.length).fill(1), ...Array(L - it.ids.length).fill(0)]); diff --git a/laya-ts/src/dx.ts b/laya-ts/src/dx.ts new file mode 100644 index 000000000..7bb40e1f0 --- /dev/null +++ b/laya-ts/src/dx.ts @@ -0,0 +1,47 @@ +import type { QuestionDef, SystemAnswer } from "./agent.js"; + +/** One-line question factories; output always passes checkQuestion. */ +export const q = { + choice(instructions: string, criteria: Record | string[]): QuestionDef { + return { type: "choice", instructions, criteria } as QuestionDef; + }, + score(instructions: string, criteria: unknown[]): QuestionDef { + return { type: "score", instructions, criteria } as QuestionDef; + }, + noul(instructions: string, criteria?: Record, labels?: { false: string; true: string }): QuestionDef { + const out: Record = { type: "noul", instructions }; + if (criteria !== undefined) out["criteria"] = criteria; + if (labels !== undefined) out["labels"] = labels; + return out as QuestionDef; + }, +}; + +/** Argmax label: choice string, score key, noul boolean. */ +export function best(answer: SystemAnswer): string | boolean { + if (answer.type === "choice") return answer.choice; + if (answer.type === "score") { + let bk = ""; + let bv = -Infinity; + for (const [k, v] of Object.entries(answer.probabilities)) { + if (v > bv) { + bv = v; + bk = k; + } + } + return bk; + } + return answer.noul >= 0.5; +} + +/** Top-k probability keys, descending. */ +export function topK(probs: Record, k: number): string[] { + return Object.entries(probs) + .sort((a, b) => b[1] - a[1]) + .slice(0, Math.max(0, k)) + .map(([key]) => key); +} + +/** Confidence gate (default 0.8). */ +export function isConfident(answer: Pick, thresh = 0.8): boolean { + return (answer.confidence ?? 0) >= thresh; +} diff --git a/laya-ts/src/hooks.ts b/laya-ts/src/hooks.ts index 8d5428acf..cbd198a2f 100644 --- a/laya-ts/src/hooks.ts +++ b/laya-ts/src/hooks.ts @@ -231,9 +231,18 @@ export function composeHooks( onPredictStart?: PredictHookArg, onPredictEnd?: PredictHookArg, ): Hook[] { + if ( + defaultHooksList.length === 0 && installed.length === 0 && (hooks === null || hooks === undefined) && + (onPredictStart === null || onPredictStart === undefined) && + (onPredictEnd === null || onPredictEnd === undefined) + ) { + return EMPTY_HOOKS; + } return [...defaultHooksList, ...installed, ...normaliseHooks(hooks, onPredictStart, onPredictEnd)]; } +const EMPTY_HOOKS: Hook[] = []; + /** * Base class giving a runtime-mutable hook list. * diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts index 6f81120b0..645e6132c 100644 --- a/laya-ts/src/index.ts +++ b/laya-ts/src/index.ts @@ -53,5 +53,7 @@ export { TEMP_MAX, } from "./common.js"; export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix } from "./common.js"; +export { maxOf } from "./common.js"; +export { q, best, topK, isConfident } from "./dx.js"; export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js"; export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js"; diff --git a/laya-ts/src/providers.ts b/laya-ts/src/providers.ts index 01670908b..c55491b6d 100644 --- a/laya-ts/src/providers.ts +++ b/laya-ts/src/providers.ts @@ -15,27 +15,47 @@ export interface SessionProvider { } function toNested(data: ArrayLike, dims: number[]): any { - const flat = Array.from(data as any, (v: any) => (typeof v === "bigint" ? Number(v) : v)); + // Single-pass copy + precomputed steps; no per-node slice/reduce. + let total = 1; + for (const d of dims) total *= d; + const flat = new Array(total); + for (let i = 0; i < total; i++) { + const v = (data as any)[i]; + flat[i] = typeof v === "bigint" ? Number(v) : v; + } if (dims.length === 0) return flat[0]; + const steps: number[] = new Array(dims.length); + for (let d = 0; d < dims.length; d++) { + let s = 1; + for (let k = d + 1; k < dims.length; k++) s *= dims[k]; + steps[d] = s; + } const rec = (d: number, off: number): any => { if (d === dims.length - 1) return flat.slice(off, off + dims[d]); - const step = dims.slice(d + 1).reduce((a, b) => a * b, 1); - const out: any[] = []; - for (let i = 0; i < dims[d]; i++) out.push(rec(d + 1, off + i * step)); + const out: any[] = new Array(dims[d]); + for (let i = 0; i < dims[d]; i++) out[i] = rec(d + 1, off + i * steps[d]); return out; }; return rec(0, 0); } function i64(ort: any, arr: number[] | number[][], dims: number[]): any { - const flat = (arr as any).flat(Infinity).map((v: number) => BigInt(Math.trunc(v))); - return new ort.Tensor("int64", BigInt64Array.from(flat), dims); + // Rank <= 2 by construction; direct loop avoids flat(Infinity) intermediates. + const out = new BigInt64Array(dims.reduce((a, b) => a * b, 1)); + let p = 0; + if (Array.isArray((arr as any)[0])) { + for (const row of arr as number[][]) for (const v of row) out[p++] = BigInt(Math.trunc(v)); + } else { + for (const v of arr as number[]) out[p++] = BigInt(Math.trunc(v)); + } + return new ort.Tensor("int64", out, dims); } /** Encoder feeds: input_ids + attention_mask (int64). */ export function feed(ort: any, b: Batch): Record { const n = b.inputIds.length; - const L = Math.max(1, ...b.inputIds.map((r) => r.length)); + let L = 1; + for (const r of b.inputIds) if (r.length > L) L = r.length; return { input_ids: i64(ort, b.inputIds, [n, L]), attention_mask: i64(ort, b.attentionMask, [n, L]), @@ -45,13 +65,22 @@ export function feed(ort: any, b: Batch): Record { /** Head feeds: encoder hidden + marker_pos/mask + qtype. */ export function feedHead(ort: any, hidden: number[][][] | any, b: Batch): Record { const n = b.markerPos.length; - const k = Math.max(1, ...b.markerPos.map((r) => r.length)); - const H = new ort.Tensor( - "float32", - Float32Array.from((hidden as any).flat(Infinity).map(Number)), - [(hidden as any).length ?? n, (hidden as any)[0]?.length ?? 1, (hidden as any)[0]?.[0]?.length ?? 1], - ); + let k = 1; + for (const r of b.markerPos) if (r.length > k) k = r.length; + // Direct flatten into typed arrays; no flat(Infinity)+map intermediates. + const nH = (hidden as any).length ?? n; const S = (hidden as any)[0]?.length ?? 1; + const Hd = (hidden as any)[0]?.[0]?.length ?? 1; + const flatH = new Float32Array(nH * S * Hd); + let p = 0; + for (let i = 0; i < nH; i++) { + const bi = (hidden as any)[i] ?? []; + for (let j = 0; j < S; j++) { + const hj = bi[j] ?? []; + for (let h = 0; h < Hd; h++) flatH[p++] = Number(hj[h] ?? 0); + } + } + const H = new ort.Tensor("float32", flatH, [nH, S, Hd]); // Pad/trim mask rows to S so the mask always matches hidden_states even // if a caller passes unpadded rows. const maskRows = b.attentionMask.map((r) => { @@ -62,11 +91,13 @@ export function feedHead(ort: any, hidden: number[][][] | any, b: Batch): Record return { hidden_states: H, marker_pos: i64(ort, b.markerPos, [n, k]), - marker_mask: new ort.Tensor( - "bool", - Uint8Array.from((b.markerMask as any).flat(Infinity).map((v: any) => (v ? 1 : 0))), - [n, k], - ), + marker_mask: new ort.Tensor("bool", (() => { + const out = new Uint8Array(n * k); + let q = 0; + for (const row of b.markerMask as unknown as boolean[][]) + for (let j = 0; j < k; j++) out[q++] = row[j] ? 1 : 0; + return out; + })(), [n, k]), qtype: i64(ort, b.qtype.map((v) => [v]), [n, 1]), // Padding mask for the head transformer (py DecisionModel.forward). // Without it, batch mates of unequal length corrupt each other's markers. @@ -103,15 +134,54 @@ function isOomError(e: unknown): boolean { return m.includes("memory") || m.includes("cuda") || m.includes("out of memory") || m.includes("oom"); } +export interface LoadOptions { + subfolder?: string | null; + signal?: AbortSignal | null; + onProgress?: ((done: number, total: number, file: string) => void) | null; +} + +const CACHE_KEY = "laya-ts-v1"; +const TOKENIZER_CANDIDATES = ["tokenizer.json", "tokenizer/tokenizer.json"]; + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +/** fetch with 2 retries on network errors + 429/5xx; 404s fail fast. */ +async function fetchWithRetry(url: string, init?: RequestInit, retries = 2): Promise { + let last: unknown = null; + for (let attempt = 0; attempt <= retries; attempt++) { + if (init?.signal?.aborted) throw new DOMException("aborted", "AbortError"); + try { + const res = await fetch(url, init); + if (res.ok) return res; + if ((res.status === 429 || res.status >= 500) && attempt < retries) { + await sleep(100 * (attempt + 1)); + continue; + } + return res; + } catch (e) { + last = e; + if ((e as any)?.name === "AbortError") throw e; + if (attempt < retries) { + await sleep(100 * (attempt + 1)); + continue; + } + throw e; + } + } + throw last instanceof Error ? last : new Error(`fetch failed for ${url}`); +} + /** Online-first fetch: try network, cache on success, fall back to CacheStorage. */ -async function fetchArrayBuffer(url: string): Promise { +async function fetchArrayBuffer(url: string, opts?: { signal?: AbortSignal | null }): Promise { const g = globalThis as unknown as { caches?: any }; let cache: any = null; let hit: any = null; try { if (g.caches && typeof g.caches.open === "function") { try { - cache = await g.caches.open("laya-ts"); + cache = await g.caches.open(CACHE_KEY); try { hit = await cache.match(url); } catch { @@ -126,7 +196,7 @@ async function fetchArrayBuffer(url: string): Promise { } if (cache) { try { - const res = await fetch(url); + const res = await fetchWithRetry(url, { signal: opts?.signal ?? undefined }); if (res.ok) { try { await cache.put(url, res.clone()); @@ -154,13 +224,13 @@ async function fetchArrayBuffer(url: string): Promise { } throw new Error(`fetch failed for ${url}`); } - const res = await fetch(url); + const res = await fetchWithRetry(url, { signal: opts?.signal ?? undefined }); if (!res.ok) throw new Error(`fetch failed for ${url}: ${res.status}`); return await res.arrayBuffer(); } -async function fetchJson(url: string): Promise { - const buf = await fetchArrayBuffer(url); +async function fetchJson(url: string, opts?: { signal?: AbortSignal | null }): Promise { + const buf = await fetchArrayBuffer(url, opts); return JSON.parse(new TextDecoder().decode(buf)); } @@ -172,7 +242,7 @@ export interface NodeBundle { export async function loadNodeBundle( modelDirOrRepo: string, - opts?: { subfolder?: string | null; localDir?: string; token?: string | null }, + opts?: { subfolder?: string | null; localDir?: string; token?: string | null; signal?: AbortSignal | null; onProgress?: ((done: number, total: number, file: string) => void) | null }, ): Promise { const fs: typeof import("node:fs/promises") = await import("node:fs/promises"); const path: typeof import("node:path") = await import("node:path"); @@ -195,12 +265,20 @@ export async function loadNodeBundle( await fs.mkdir(cache, { recursive: true }); const token = opts?.token ?? (typeof process !== "undefined" ? (process as any).env?.["HF_TOKEN"] : undefined); - for (const f of ["rl_agent_config.json", "tokenizer.json", "tokenizer/tokenizer.json", "encoder.onnx", "head.onnx"]) { + const files = ["rl_agent_config.json", "tokenizer.json", "tokenizer/tokenizer.json", "encoder.onnx", "head.onnx"]; + let done = 0; + for (const f of files) { try { await fs.stat(path.join(cache, f)); } catch { const url = `https://huggingface.co/${modelDirOrRepo}/resolve/main/${sub ? sub + "/" : ""}${f}`; - const res = await fetch(url, token ? { headers: { Authorization: `Bearer ${token}` } } : undefined); + const res = await fetchWithRetry( + url, + { + ...(token ? { headers: { Authorization: `Bearer ${token}` } } : {}), + signal: opts?.signal ?? undefined, + }, + ); if (!res.ok) { if (f === "rl_agent_config.json") { throw new Error( @@ -213,6 +291,8 @@ export async function loadNodeBundle( await fs.mkdir(path.dirname(target), { recursive: true }); await fs.writeFile(target, new Uint8Array(await res.arrayBuffer())); } + done++; + opts?.onProgress?.(done, files.length, f); } dir = cache; } @@ -225,7 +305,7 @@ export async function loadNodeBundle( ); } let tokenizerJson: unknown | null = null; - for (const candidate of ["tokenizer.json", "tokenizer/tokenizer.json"]) { + for (const candidate of TOKENIZER_CANDIDATES) { try { tokenizerJson = JSON.parse(await fs.readFile(path.join(dir, candidate), "utf8")); break; @@ -252,24 +332,27 @@ function baseUrlFor(repoOrUrl: string, subfolder?: string | null): string { export async function loadWebBundle( repoOrUrl: string, - opts?: { subfolder?: string | null }, + opts?: { subfolder?: string | null; signal?: AbortSignal | null; onProgress?: ((done: number, total: number, file: string) => void) | null }, ): Promise { const base = baseUrlFor(repoOrUrl, opts?.subfolder ?? null); + const sig = { signal: opts?.signal ?? undefined }; let cfg: any; try { - cfg = await fetchJson(`${base}/rl_agent_config.json`); + cfg = await fetchJson(`${base}/rl_agent_config.json`, sig); } catch { throw new Error(`Incompatible model: ${JSON.stringify(repoOrUrl)} does not contain 'rl_agent_config.json'.`); } + opts?.onProgress?.(1, 2, "rl_agent_config.json"); let tokenizerJson: unknown | null = null; - for (const candidate of ["tokenizer.json", "tokenizer/tokenizer.json"]) { + for (const candidate of TOKENIZER_CANDIDATES) { try { - tokenizerJson = await fetchJson(`${base}/${candidate}`); + tokenizerJson = await fetchJson(`${base}/${candidate}`, sig); break; } catch { // Try the next supported Hugging Face layout. } } + opts?.onProgress?.(2, 2, "tokenizer.json"); return { dir: base, cfg, tokenizerJson }; } diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts index b98941bad..0abc80717 100644 --- a/laya-ts/src/router.ts +++ b/laya-ts/src/router.ts @@ -495,4 +495,100 @@ export class Router extends HookRegistry { ): Promise { return this.predict(state, questions, opts); } + + /** + * Route each state, then answer each model group in one forward pass. + * + * Same questions for every state. Loads each checkpoint once, keeps input + * order, and attaches per-state `routing`. Agents without predictMany fall + * back to looped systemOne. + */ + async predictMany( + states: unknown[], + questions: Record, + opts: RouteOptions & PredictOptions = {}, + ): Promise { + const input = [...(states ?? [])]; + if (input.length === 0) return []; + const active = composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd); + const raiseErrors = opts.hooksRaise ?? this.hooksRaise; + let decisions = input.map((s) => this.route(s, questions, opts)); + const ctx = new PredictContext({ + states: [...input], + questions: questions as Record, + router: this, + }); + try { + dispatch(active, "onPredictStart", ctx, { raiseErrors }); + if (ctx.results === null) { + if (ctx.states.length !== decisions.length) { + decisions = ctx.states.map((s) => this.route(s, ctx.questions as Record, opts)); + } + const out = new Array(ctx.states.length); + const groups = new Map(); + decisions.forEach((d, i) => { + const list = groups.get(d.model) ?? []; + list.push(i); + groups.set(d.model, list); + }); + for (const [model, idxs] of groups) { + const agent = (await this.load(model)) as { + predictMany?(s: unknown[], q: Record): Promise; + systemMany?(s: unknown[], q: Record): Promise; + systemOne(s: unknown, q: Record): Promise; + }; + const groupStates = idxs.map((i) => ctx.states[i]); + let results: SystemOneResult[]; + if (typeof agent.predictMany === "function") { + results = await agent.predictMany(groupStates, ctx.questions as Record); + } else if (typeof agent.systemMany === "function") { + results = await agent.systemMany(groupStates, ctx.questions as Record); + } else { + results = []; + for (const s of groupStates) { + results.push(await agent.systemOne(s, ctx.questions as Record)); + } + } + idxs.forEach((gi, k) => { + (out[gi] as RoutedResult) = { ...(results[k] as RoutedResult), routing: { ...decisions[gi] } }; + }); + } + ctx.results = out as unknown as Record[]; + ctx.model ??= decisions[0]?.model ?? null; + } else { + for (let i = 0; i < ctx.results.length; i++) { + const result = ctx.results[i]; + if (result && typeof result === "object" && !("routing" in result) && decisions[i]) { + (result as unknown as RoutedResult).routing = { ...decisions[i] }; + } + } + } + } catch (err) { + ctx.error = err; + try { + dispatch(active, "onError", ctx, { raiseErrors }); + } catch { + // A failing onError hook must not hide the failure that triggered it. + } + throw err; + } finally { + ctx.markElapsed(); + if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results); + try { + dispatch(active, "onPredictEnd", ctx, { raiseErrors }); + } catch (hookErr) { + // End hooks run on the failure path too; do not let one mask the real error. + if (ctx.error === null) throw hookErr; + } + } + return ctx.results as unknown as RoutedResult[]; + } + + async systemMany( + states: unknown[], + questions: Record, + opts: RouteOptions & PredictOptions = {}, + ): Promise { + return this.predictMany(states, questions, opts); + } } diff --git a/laya-ts/src/shortlist.ts b/laya-ts/src/shortlist.ts index 89b8d0f0e..f856ae96c 100644 --- a/laya-ts/src/shortlist.ts +++ b/laya-ts/src/shortlist.ts @@ -1,4 +1,4 @@ -import { renderOptions, serializeState } from "./common.js"; +import { maxOf, renderOptions, serializeState } from "./common.js"; import type { Batch } from "./providers.js"; import type { QuestionDef, SystemOneResult } from "./agent.js"; @@ -130,7 +130,7 @@ export async function predictShortlist( agent: unknown, state: unknown, questions: Record, - embedFn: EmbedFn, + embedFn?: EmbedFn | null, k: number = DEFAULT_SHORTLIST_K, predictKwargs: Record = {}, ): Promise }> { @@ -138,6 +138,15 @@ export async function predictShortlist( throw new TypeError("questions must be a dict of question id -> definition"); } const checked = checkK(k); + const needsEmbed = Object.values(questions).some( + (qdef) => + typeof qdef === "object" && qdef !== null && !Array.isArray(qdef) && + (qdef as Record)["type"] === "choice" && + "criteria" in (qdef as Record) && + Object.keys((qdef as Record)["criteria"] as object ?? {}).length > checked, + ); + // Default bi-encoder is the agent itself; pass a dedicated embedFn only when it shortlists better. + const embed = embedFn ?? (needsEmbed ? embedFnFromAgent(agent) : null); const reduced: Record = {}; const meta: Record = {}; for (const [qid, qdef] of Object.entries(questions)) { @@ -149,7 +158,7 @@ export async function predictShortlist( throw new Error(`question ${JSON.stringify(qid)} is a choice but has no criteria`); } const q = qdef as Record; - const { labels, scores, passthrough, n } = await rank(state, q["criteria"], embedFn, checked, q["instructions"]); + const { labels, scores, passthrough, n } = await rank(state, q["criteria"], embed as EmbedFn, checked, q["instructions"]); meta[qid] = { labels: [...labels], scores, k: checked, n, passthrough }; if (passthrough) { reduced[qid] = qdef; @@ -189,7 +198,7 @@ export function embedFnFromAgent(agent: unknown, maxLength = 512, batchSize = 32 for (let s = 0; s < rows.length; s += batchSize) { const chunk = rows.slice(s, s + batchSize); const ids = chunk.map((t) => tok.encode(t).slice(0, maxLength)); - const L = Math.max(1, ...ids.map((r) => r.length)); + const L = maxOf(ids.map((r) => r.length), 1); const pad = tok.padId ?? 0; const batch: Batch = { inputIds: ids.map((r) => [...r, ...Array(L - r.length).fill(pad)]), diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.ts index 24cfe8f76..a6018c725 100644 --- a/laya-ts/src/tokenizer.ts +++ b/laya-ts/src/tokenizer.ts @@ -58,18 +58,31 @@ function maps(): { b2u: Map; u2b: Map } { const GPT2_SPLIT = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu; /** Merge one word's chars greedily by merge rank (first occurrence per step). */ +// Word cache per merges map (cap 20k, clear on overflow); upgrade to LRU if hit rate matters. +const wordCache = new WeakMap, Map>(); function bpeWord(chars: string[], rank: Map): string[] { + if (chars.length <= 1) return chars.slice(); + let per = wordCache.get(rank); + if (!per) { + per = new Map(); + wordCache.set(rank, per); + } + const key = chars.length + ":" + chars.join(""); + const hit = per.get(key); + if (hit) return hit.slice(); let word = chars.slice(); - if (word.length <= 1) return word; for (;;) { let best = Infinity, idx = -1; for (let i = 0; i < word.length - 1; i++) { const r = rank.get(word[i] + " " + word[i + 1]); if (r !== undefined && r < best) { best = r; idx = i; } } - if (idx < 0) return word; + if (idx < 0) break; word = [...word.slice(0, idx), word[idx] + word[idx + 1], ...word.slice(idx + 2)]; } + if (per.size > 20000) per.clear(); + per.set(key, word.slice()); + return word; } /** Byte-level BPE encode: NFC-normalize, NO lowercasing, GPT-2 byte map + rank-order merges. */ diff --git a/laya-ts/tests/batch.test.ts b/laya-ts/tests/batch.test.ts new file mode 100644 index 000000000..d9f149cf0 --- /dev/null +++ b/laya-ts/tests/batch.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { Agent } from "../src/agent.js"; +import { Router } from "../src/router.js"; + +function fakeProvider(counter: { enc: number }) { + return { + async runEncoder(batch: any) { + counter.enc++; + return { lastHidden: batch.inputIds.map((row: number[]) => row.map(() => [1, 0])) }; + }, + async runHead(_h: unknown, batch: any) { + return { + logits: batch.markerPos.map((m: number[]) => m.map((_, i) => 2 - i * 0.5)), + act: batch.markerPos.map(() => [3, 0]), + }; + }, + }; +} + +const Q: any = { + d: { type: "choice", instructions: "q?", criteria: { x: "yes", y: "no" } }, +}; + +describe("predictMany", () => { + it("batches N states into one encoder run with per-state results", async () => { + const counter = { enc: 0 }; + const a = new Agent({ provider: fakeProvider(counter) } as any); + const out = await (a as any).predictMany(["hi one", "hi two"], Q); + expect(out).toHaveLength(2); + expect(out[0].answers.d.choice).toBe("x"); + expect(out[1].answers.d.choice).toBe("x"); + expect(counter.enc).toBe(1); + }); + it("matches looped systemOne", async () => { + const c1 = { enc: 0 }; + const c2 = { enc: 0 }; + const a1 = new Agent({ provider: fakeProvider(c1) } as any); + const a2 = new Agent({ provider: fakeProvider(c2) } as any); + const states = ["alpha", "beta", "gamma"]; + const batched = await (a1 as any).predictMany(states, Q); + const looped = []; + for (const s of states) looped.push(await a2.systemOne(s, Q)); + expect(batched.map((r: any) => r.answers)).toEqual(looped.map((r) => r.answers)); + }); + it("empty states and empty questions", async () => { + const a = new Agent({ provider: fakeProvider({ enc: 0 }) } as any); + expect(await (a as any).predictMany([], Q)).toEqual([]); + const out = await (a as any).predictMany(["hi"], {}); + expect(out[0].answers).toEqual({}); + }); +}); + +describe("router predictMany", () => { + function loaderWithCounts(calls: Record) { + return async (name: string) => { + calls[name] = (calls[name] ?? 0) + 1; + const agent = new Agent({ provider: fakeProvider({ enc: 0 }) } as any); + return agent; + }; + } + it("routes each state, loads once per model, keeps order + routing keys", async () => { + const calls: Record = {}; + const r = new Router({ loader: loaderWithCounts(calls) } as any); + const states = [ + "Please refund the duplicate charge", + "मुझसे दो बार शुल्क लिया गया", + "Please refund again", + ]; + const out = await (r as any).predictMany(states, Q); + expect(out).toHaveLength(3); + expect(out[0].routing.model).toBe("english"); + expect(out[1].routing.model).toBe("multilingual"); + expect(out[2].routing.model).toBe("english"); + expect(calls["english"]).toBe(1); + expect(calls["multilingual"]).toBe(1); + }); + it("matches looped predict answers", async () => { + const r1 = new Router({ loader: loaderWithCounts({}) } as any); + const r2 = new Router({ loader: loaderWithCounts({}) } as any); + const states = ["hello there", "Der Kunde wurde zweimal belastet"]; + const batched = await (r1 as any).predictMany(states, Q); + const looped = []; + for (const s of states) looped.push(await r2.predict(s, Q)); + expect(batched.map((x: any) => x.answers)).toEqual(looped.map((x) => x.answers)); + expect(batched.map((x: any) => x.routing.model)).toEqual(looped.map((x) => x.routing.model)); + }); +}); diff --git a/laya-ts/tests/dx.test.ts b/laya-ts/tests/dx.test.ts new file mode 100644 index 000000000..60c5536e3 --- /dev/null +++ b/laya-ts/tests/dx.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { checkQuestion } from "../src/agent.js"; +import { q, best, topK, isConfident } from "../src/dx.js"; +import { predictShortlist } from "../src/shortlist.js"; + +describe("q builders", () => { + it("produce valid defs", () => { + const c: any = q.choice("What?", { a: "x", b: "y" }); + const s: any = q.score("Rate?", ["bad", "ok"]); + const n: any = q.noul("Holds?"); + expect(() => checkQuestion("c", c)).not.toThrow(); + expect(() => checkQuestion("s", s)).not.toThrow(); + expect(() => checkQuestion("n", n)).not.toThrow(); + expect(c.criteria).toEqual({ a: "x", b: "y" }); + }); + it("choice accepts label list", () => { + const c: any = q.choice("Pick?", ["a", "b"]); + expect(() => checkQuestion("c", c)).not.toThrow(); + }); +}); + +describe("result helpers", () => { + it("best/topK/isConfident", () => { + expect(best({ type: "choice", choice: "b", probabilities: { a: 0.2, b: 0.8 } } as any)).toBe("b"); + expect(best({ type: "score", probabilities: { "0": 0.1, "1": 0.7, "2": 0.2 } } as any)).toBe("1"); + expect(topK({ a: 0.2, b: 0.8, c: 0.5 }, 2)).toEqual(["b", "c"]); + expect(isConfident({ confidence: 0.9 } as any, 0.8)).toBe(true); + expect(isConfident({ confidence: 0.5 } as any, 0.8)).toBe(false); + }); +}); + +describe("predictShortlist auto-embed", () => { + it("uses agent encoder when embedFn omitted", async () => { + const agent: any = { + tok: { encode: (t: string) => [t.length % 100], padId: 0 }, + provider: { + async runEncoder(batch: any) { + return { lastHidden: batch.inputIds.map((row: number[]) => row.map(() => [1, 0])) }; + }, + }, + async predict(_s: unknown, reduced: any) { + return { model: "m", answers: {}, usage: { input_tokens: 0, output_tokens: 0 }, _reduced: reduced }; + }, + }; + const questions: any = { + d: { type: "choice", instructions: "pick", criteria: { aaa: "x", bbb: "y", ccc: "z" } }, + }; + const out: any = await predictShortlist(agent, "state", questions, undefined as any, 1); + expect(out.shortlist.d.labels).toHaveLength(1); + }); +}); diff --git a/laya-ts/tests/flat-parity.test.ts b/laya-ts/tests/flat-parity.test.ts new file mode 100644 index 000000000..7c5ab3230 --- /dev/null +++ b/laya-ts/tests/flat-parity.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { feed, feedHead } from "../src/providers.js"; + +const ort = { + Tensor: class { + type: string; + data: any; + dims: number[]; + constructor(type: string, data: any, dims: number[]) { + this.type = type; + this.data = data; + this.dims = dims; + } + }, +}; + +describe("flat parity", () => { + it("feedHead flattens nested hidden without value drift", () => { + const hidden = [ + [ + [1, 2], + [3, 4], + ], + ]; + const batch: any = { + inputIds: [[7, 8]], + attentionMask: [[1, 1]], + markerPos: [[0, 1]], + markerMask: [[true, true]], + qtype: [0], + }; + const out = feedHead(ort, hidden, batch); + expect(out.hidden_states.dims).toEqual([1, 2, 2]); + expect(Array.from(out.hidden_states.data)).toEqual([1, 2, 3, 4]); + }); + it("feed builds int64 ids+mask", () => { + const out = feed(ort, { + inputIds: [[5, 6]], + attentionMask: [[1, 1]], + markerPos: [[0]], + markerMask: [[true]], + qtype: [0], + } as any); + expect(out.input_ids.dims).toEqual([1, 2]); + expect(Array.from(out.input_ids.data)).toEqual([5n, 6n]); + }); +}); diff --git a/laya-ts/tests/load-robustness.test.ts b/laya-ts/tests/load-robustness.test.ts new file mode 100644 index 000000000..7526b2b05 --- /dev/null +++ b/laya-ts/tests/load-robustness.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi, afterEach } from "vitest"; +import { loadWebBundle } from "../src/providers.js"; + +afterEach(() => vi.unstubAllGlobals()); + +function jsonResponse(obj: unknown, ok = true, status = 200) { + const text = JSON.stringify(obj); + const buf = new TextEncoder().encode(text).buffer as ArrayBuffer; + return { + ok, + status, + clone: () => jsonResponse(obj, ok, status), + arrayBuffer: async () => buf.slice(0), + }; +} + +describe("load robustness", () => { + it("retries transient 503 then succeeds", async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce({ ok: false, status: 503, arrayBuffer: async () => new ArrayBuffer(0), clone: () => ({}) }) + .mockResolvedValueOnce(jsonResponse({ max_len: 512 })) + .mockResolvedValueOnce(jsonResponse({ model: { vocab: {}, merges: [] }, added_tokens: [] })); + vi.stubGlobal("fetch", fetchMock); + const progress: string[] = []; + const bundle = await loadWebBundle("https://example.com/m", { + onProgress: (_d: number, _t: number, f: string) => progress.push(f), + } as any); + expect(bundle.cfg).toEqual({ max_len: 512 }); + expect(fetchMock.mock.calls.length).toBeGreaterThanOrEqual(3); + expect(progress.length).toBeGreaterThan(0); + }); + it("aborted signal throws without retry", async () => { + const c = new AbortController(); + c.abort(); + vi.stubGlobal("fetch", vi.fn(async (_u: any, init: any) => { + if (init?.signal?.aborted ?? c.signal.aborted) throw new DOMException("aborted", "AbortError"); + return jsonResponse({}); + })); + await expect(loadWebBundle("https://example.com/m", { signal: c.signal } as any)).rejects.toThrow(); + }); +}); diff --git a/laya-ts/tests/maxlen.test.ts b/laya-ts/tests/maxlen.test.ts new file mode 100644 index 000000000..1be38e8f4 --- /dev/null +++ b/laya-ts/tests/maxlen.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { maxOf } from "../src/common.js"; + +describe("maxOf", () => { + it("returns max without spread overflow", () => { + expect(maxOf([1, 5, 3], 1)).toBe(5); + expect(maxOf([], 1)).toBe(1); + }); + it("handles 300k rows without RangeError", () => { + const lens = new Array(300_000).fill(1); + lens[123456] = 77; + expect(maxOf(lens, 1)).toBe(77); + }); +}); diff --git a/laya-ts/tests/prefix-cache.test.ts b/laya-ts/tests/prefix-cache.test.ts new file mode 100644 index 000000000..503fa4759 --- /dev/null +++ b/laya-ts/tests/prefix-cache.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { buildQuestionPrefix } from "../src/common.js"; +import { defaultTokenizer } from "../src/agent.js"; + +describe("prefix cache", () => { + it("returns cached prefix on repeat", () => { + const tok = defaultTokenizer(); + const q: any = { t: "choice", ins: "What?", crit: { a: "x", b: "y" } }; + const p1 = buildQuestionPrefix(tok, q, 512, 192); + const p2 = buildQuestionPrefix(tok, q, 512, 192); + expect(p2).toBe(p1); + expect(p2.ids).toEqual(p1.ids); + }); +});