Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions laya-ts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,19 @@ const out = await predictShortlist(agent, state, questions, embedFn, 20);

```bash
node laya-ts/examples/try-ml.mjs # needs ./model-ml from the export step
node laya-ts/examples/try-batch.mjs # bulk predictMany over 3 states, one forward pass per model
node laya-ts/examples/snake.mjs --ticks 50 # autonomous snake demo, headless smoke (live TUI without --ticks)
```

## Batch (one forward pass for N states)

```ts
const out = await agent.predictMany(states, questions); // SystemOneResult[]
const routed = await router.predictMany(states, questions); // + routing per state
import { q, best, topK } from "laya-ts";
const qq = { d: q.choice("pick", { refund: "money back", other: "rest" }) };
```

## Packaging

ponytail: CJS/browser-field dual build + tsconfig tests-include deferred — Task 7 verified ESM-only; CJS needs second tsc config + export-map change, untested. Add when a CJS consumer or browser-field swap is requested.
3 changes: 2 additions & 1 deletion laya-ts/examples/snake.mjs
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import { Agent } from "../dist/index.js";
import { existsSync } from "node:fs";

// ---- CLI (no deps) ----
const args = process.argv.slice(2);
const opt = (name, def) => {
const i = args.indexOf(name);
return i === -1 || i + 1 >= args.length ? def : args[i + 1];
};
const MODEL_DIR = opt("--model", "./model-ml");
const MODEL_DIR = opt("--model", ["./model-ml", "laya-ts/examples/model-ml", "./examples/model-ml"].find((d) => existsSync(d)) ?? "./model-ml");
const TICK_MS = Math.max(20, Number(opt("--tick", "120")) || 120);
const W = Math.max(10, Number(opt("--width", "20")) || 20);
const H = Math.max(6, Number(opt("--height", "12")) || 12);
Expand Down
43 changes: 43 additions & 0 deletions laya-ts/examples/try-batch.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { Agent, Router } from "../dist/index.js";
import { existsSync } from "node:fs";

// run as: node laya-ts/examples/try-batch.mjs (repo root) or node examples/try-batch.mjs (laya-ts dir)
// --model overrides; otherwise first existing dir wins (repo ships weights at laya-ts/examples/model-ml)
const args = process.argv.slice(2);
const flag = (name) => {
const i = args.indexOf(name);
return i === -1 || i + 1 >= args.length ? null : args[i + 1];
};
const MODEL_DIR = flag("--model") ??
["./model-ml", "laya-ts/examples/model-ml", "./examples/model-ml"].find((d) => existsSync(d)) ?? "./model-ml";
const agent = await Agent.load(MODEL_DIR);

const states = [
{ body: "charged twice, refund please" },
{ body: "मुझसे दो बार शुल्क लिया गया" },
{ body: "Der Kunde wurde zweimal belastet" },
];
const questions = {
department: {
type: "choice",
instructions: "Which department should handle this request?",
criteria: { billing: "invoices, payments, refunds", technical: "bugs, outages", other: "rest" },
},
};

// One forward pass for all states (vs N sequential predicts).
const batched = await agent.predictMany(states, questions);
for (const [i, r] of batched.entries()) {
const a = r.answers.department;
console.log(`batched[${i}]:`, a?.choice, `conf=${a?.confidence}`, JSON.stringify(a?.probabilities));
}

// Router groups by language, one load per checkpoint, order preserved.
// Same local weights answer both groups so the demo runs offline.
const router = new Router();
router.attach("english", agent);
router.attach("multilingual", agent);
const routed = await router.predictMany(states, questions);
for (const [i, r] of routed.entries()) {
console.log(`routed[${i}]:`, r.routing.model, "->", r.answers.department?.choice);
}
12 changes: 11 additions & 1 deletion laya-ts/examples/try-ml.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
import { Agent, Router } from "../dist/index.js";
import { existsSync } from "node:fs";

const agent = await Agent.load("./model-ml"); // run as: node laya-ts/examples/try-ml.mjs (repo root)
// run as: node laya-ts/examples/try-ml.mjs (repo root) or node examples/try-ml.mjs (laya-ts dir)
// --model overrides; otherwise first existing dir wins (repo ships weights at laya-ts/examples/model-ml)
const args = process.argv.slice(2);
const flag = (name) => {
const i = args.indexOf(name);
return i === -1 || i + 1 >= args.length ? null : args[i + 1];
};
const MODEL_DIR = flag("--model") ??
["./model-ml", "laya-ts/examples/model-ml", "./examples/model-ml"].find((d) => existsSync(d)) ?? "./model-ml";
const agent = await Agent.load(MODEL_DIR);
console.log("loaded:", agent.cfg.encoder ?? "multilingual");

// Direct predict (Hindi -> multilingual weights)
Expand Down
186 changes: 116 additions & 70 deletions laya-ts/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,28 @@ export class Agent extends HookRegistry {
return (await this._predictHooked([state], questions, opts))[0];
}

/**
* Evaluate typed questions across many states in a single forward pass.
*
* Same questions for every state (bulk triage). One encoder+head run for
* all S*Q rows instead of S sequential runs. Hooks see all states at once.
*/
async predictMany(
states: unknown[],
questions: Record<string, QuestionDef>,
opts: PredictOptions = {},
): Promise<SystemOneResult[]> {
return this._predictHooked([...(states ?? [])], questions, opts);
}

async systemMany(
states: unknown[],
questions: Record<string, QuestionDef>,
opts: PredictOptions = {},
): Promise<SystemOneResult[]> {
return this.predictMany(states, questions, opts);
}

private async _predictHooked(
states: unknown[],
questions: Record<string, QuestionDef>,
Expand All @@ -315,10 +337,7 @@ export class Agent extends HookRegistry {
try {
dispatch(active, "onPredictStart", ctx, { raiseErrors });
if (ctx.results === null) {
const out: SystemOneResult[] = [];
for (const st of ctx.states) {
out.push(await this._systemOneCore(st, ctx.questions as Record<string, QuestionDef>));
}
const out = await this._systemManyCore(ctx.states, ctx.questions as Record<string, QuestionDef>);
ctx.results = out as unknown as Record<string, unknown>[];
ctx.model ??= out[0]?.model ?? null;
}
Expand Down Expand Up @@ -347,90 +366,113 @@ export class Agent extends HookRegistry {
state: unknown,
questions: Record<string, QuestionDef>,
): Promise<SystemOneResult> {
return (await this._systemManyCore([state], questions))[0];
}

// One encoder+head run for all S*Q rows; chunk upstream if GPU OOM (providers fall back to CPU).
private async _systemManyCore(
states: unknown[],
questions: Record<string, QuestionDef>,
): Promise<SystemOneResult[]> {
const ids = Object.keys(questions ?? {});
if ((states ?? []).length === 0) return [];
if (ids.length === 0) {
return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } };
return states.map(() => ({ model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } }));
}
const items: { ids: number[]; markers: number[]; qtype: number }[] = [];
const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }[] = [];
// The state text is shared by every question and is usually the longest text in the
// sequence — encode it once and compose the per-question prefixes onto it.
const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));
for (const qid of ids) {
const internals = ids.map((qid) => {
checkQuestion(qid, questions[qid]);
const q = toInternal(questions[qid]);
internals.push(q);
const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen);
const { ids: seq, markers } = sequenceWithState(
prefix, stAll, this.tok.sepId, this.maxLen, Array.isArray(state),
);
if (markers.length !== renderOptions(q).length) {
throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`);
return toInternal(questions[qid]);
});
const optCounts = internals.map((q) => renderOptions(q).length);
const prefixes = internals.map((q) => buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen));
const items: { ids: number[]; markers: number[]; qtype: number }[] = [];
const stateOfRow: number[] = [];
for (let si = 0; si < states.length; si++) {
const state = states[si];
const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));
for (let qi = 0; qi < ids.length; qi++) {
const { ids: seq, markers } = sequenceWithState(
prefixes[qi], stAll, this.tok.sepId, this.maxLen, Array.isArray(state),
);
if (markers.length !== optCounts[qi]) {
throw new Error(`question ${qidStr(ids[qi])} options exceed head_max_len=${this.headMaxLen}`);
}
items.push({ ids: seq, markers, qtype: QTYPES[internals[qi].t] });
stateOfRow.push(si);
}
items.push({ ids: seq, markers, qtype: QTYPES[q.t] });
}
const collated = collateItems([items], this.tok.padId);
if (!collated) throw new Error("no items to collate");
const batch: Batch = collated;
const nTokens = batch.attentionMask.flat().reduce((a, b) => a + b, 0);
const { lastHidden } = await this.provider.runEncoder(batch);
const { logits, act } = await this.provider.runHead(lastHidden, batch);
if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < ids.length || act.length < ids.length) {
if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < items.length || act.length < items.length) {
throw new Error("model provider returned fewer output rows than input items");
}
for (let r = 0; r < ids.length; r++) {
for (let r = 0; r < items.length; r++) {
const k = items[r].markers.length;
if (!Array.isArray(logits[r]) || logits[r].length < k || !Array.isArray(act[r]) || act[r].length === 0 ||
logits[r].some((v) => !Number.isFinite(v)) || act[r].some((v) => !Number.isFinite(v))) {
throw new Error(`model provider returned invalid output for question ${ids[r]}`);
throw new Error(`model provider returned invalid output for question ${ids[r % ids.length]}`);
}
}

const answers: Record<string, SystemAnswer> = {};
for (let r = 0; r < ids.length; r++) {
const qid = ids[r];
const q = internals[r];
const k = items[r].markers.length;
const qt = QTYPES[q.t];
const bucket = tempBucket(qt, k);
const scale = this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0;
const z = (logits[r] as number[]).slice(0, k).map((v) => v / scale);
const p = softmax(z);
const actRow = (act[r] as number[]) ?? [1, 0];
const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
const ext = { act_probability: r4(actP[0]) };
if (q.t === "choice") {
const keys = Object.keys(q.crit as Record<string, unknown>);
let best = 0;
for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i;
answers[qid] = {
type: "choice",
choice: keys[best],
probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),
confidence: r4(confidenceFromProbs(p)),
action: ext,
};
} else if (q.t === "score") {
const exp = p.reduce((a, v, i) => a + i * v, 0);
answers[qid] = {
type: "score",
score: r4(exp),
legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])),
probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),
confidence: r4(confidenceFromProbs(p)),
action: ext,
};
} else {
const pt = p[1] ?? 0;
answers[qid] = {
type: "noul",
noul: r4(pt),
confidence: r4(Math.max(pt, 1 - pt)),
action: ext,
};
const mask = batch.attentionMask;
const tokensPerState = states.map((_, si) => {
let t = 0;
for (let qi = 0; qi < ids.length; qi++) {
const row = mask[si * ids.length + qi] ?? [];
for (const v of row) t += v;
}
}
return { model: "laya-rl-agent", answers, usage: { input_tokens: nTokens, output_tokens: 0 } };
return t;
});
return states.map((_, si) => {
const answers: Record<string, SystemAnswer> = {};
for (let qi = 0; qi < ids.length; qi++) {
const r = si * ids.length + qi;
const qid = ids[qi];
const q = internals[qi];
const k = items[r].markers.length;
const qt = QTYPES[q.t];
const bucket = tempBucket(qt, k);
const scale = this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0;
const z = (logits[r] as number[]).slice(0, k).map((v) => v / scale);
const p = softmax(z);
const actRow = (act[r] as number[]) ?? [1, 0];
const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
const ext = { act_probability: r4(actP[0]) };
if (q.t === "choice") {
const keys = Object.keys(q.crit as Record<string, unknown>);
let best = 0;
for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i;
answers[qid] = {
type: "choice",
choice: keys[best],
probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),
confidence: r4(confidenceFromProbs(p)),
action: ext,
};
} else if (q.t === "score") {
const exp = p.reduce((a, v, i) => a + i * v, 0);
answers[qid] = {
type: "score",
score: r4(exp),
legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])),
probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),
confidence: r4(confidenceFromProbs(p)),
action: ext,
};
} else {
const pt = p[1] ?? 0;
answers[qid] = {
type: "noul",
noul: r4(pt),
confidence: r4(Math.max(pt, 1 - pt)),
action: ext,
};
}
}
return { model: "laya-rl-agent", answers, usage: { input_tokens: tokensPerState[si], output_tokens: 0 } };
});
}

async predict(
Expand All @@ -449,6 +491,8 @@ export class Agent extends HookRegistry {
localDir?: string;
token?: string | null;
numThreads?: number;
signal?: AbortSignal | null;
onProgress?: ((done: number, total: number, file: string) => void) | null;
},
): Promise<Agent> {
const sub = opts?.subfolder ?? null;
Expand All @@ -460,7 +504,7 @@ export class Agent extends HookRegistry {
let provider: SessionProvider;
if (isBrowser) {
const { loadWebBundle, createWebProvider } = await import("./providers.js");
const bundle = await loadWebBundle(modelDirOrRepo, { subfolder: sub });
const bundle = await loadWebBundle(modelDirOrRepo, { subfolder: sub, signal: opts?.signal, onProgress: opts?.onProgress });
cfg = bundle.cfg;
tokenizerJson = bundle.tokenizerJson;
dir = bundle.dir;
Expand All @@ -471,6 +515,8 @@ export class Agent extends HookRegistry {
subfolder: sub,
localDir: opts?.localDir,
token: opts?.token,
signal: opts?.signal,
onProgress: opts?.onProgress,
});
cfg = bundle.cfg;
tokenizerJson = bundle.tokenizerJson;
Expand Down
31 changes: 29 additions & 2 deletions laya-ts/src/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,25 @@ export interface QuestionPrefix {
}
/** The question half of `buildSequence`: everything before the state tokens. Hoisted out so
* callers asking several questions about the same state can encode the state text only once. */
// Per-tokenizer prefix cache (cap 1k, clear on overflow); repeat triage skips re-encode.
const prefixCache = new WeakMap<object, Map<string, QuestionPrefix>>();
export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,
maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix {
let per = prefixCache.get(tok as object);
if (!per) {
per = new Map();
prefixCache.set(tok as object, per);
}
const key = JSON.stringify([q.t, q.ins, q.crit, q.labels ?? null, maxLen, headMaxLen, optionOrder ?? null]);
const hit = per.get(key);
if (hit) return hit;
const built = buildQuestionPrefixUncached(tok, q, maxLen, headMaxLen, optionOrder);
if (per.size > 1000) per.clear();
per.set(key, built);
return built;
}
function buildQuestionPrefixUncached(tok: TokenizerLike, q: InternalQ,
maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix {
const maskTok = tok.maskToken;
const opts = renderOptions(q);
const order = optionOrder ?? opts.map((_, i) => i);
Expand Down Expand Up @@ -118,6 +135,12 @@ export function tempBucket(qtype: number, k: number): string {
const size = k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+";
return `${["choice", "score", "noul"][qtype]}:${size}`;
}
/** Max of a length list without spread (Math.max(...arr) throws RangeError past ~100k args). */
export function maxOf(values: ArrayLike<number>, fallback = 0): number {
let m = fallback;
for (let i = 0; i < values.length; i++) if (values[i] > m) m = values[i];
return m;
}
export interface CollateItem {
ids: number[];
markers: number[];
Expand All @@ -140,8 +163,12 @@ export interface CollatedBatch {
export function collateItems(batch: CollateItem[][], padId: number): CollatedBatch | null {
const items = (batch ?? []).flat();
if (items.length === 0) return null;
const L = Math.max(...items.map((it) => it.ids.length));
const K = Math.max(...items.map((it) => it.markers.length));
let L = 0;
let K = 0;
for (const it of items) {
if (it.ids.length > L) L = it.ids.length;
if (it.markers.length > K) K = it.markers.length;
}
const hasTarget = items.some((it) => "target" in it);
const inputIds = items.map((it) => [...it.ids, ...Array(L - it.ids.length).fill(padId)]);
const attentionMask = items.map((it) => [...Array(it.ids.length).fill(1), ...Array(L - it.ids.length).fill(0)]);
Expand Down
Loading
Loading