Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion docs/hooks/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,10 @@ router.predict(state, questions, model=None, task=None, lang=None, lang_guess=No
hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
hooks_timeout=None, max_len=None, head_max_len=None)

router.predict_batch(requests, batch_size=None, hooks_timeout=None, min_confidence=None,
sort_by_length=False, hooks=None, on_predict_start=None, on_predict_end=None,
hooks_raise=None)

router.system_one(...) # alias of predict
router.load(name) # builds on first use; fires on_load
router.preload(names=None) # builds several; fires on_load per build
Expand All @@ -304,7 +308,7 @@ router.attach(name, agent) # registers an existing agent; does not fire on_load
router.loaded # list of resident checkpoint names
```

- Per-call `hooks=` on `route` and `predict` apply to the whole call, including `on_route`.
- Per-call `hooks=` on `route`, `route_batch`, `predict` and `predict_batch` apply to the whole call, including `on_route`. On `predict_batch` the list is composed the same way as on `predict` (installed hooks first, then the per-call list; `None` and `[]` add nothing) and runs once per request.
- `route()` is public: calling it dispatches `on_route` with the installed hooks plus any
per-call `hooks`.

Expand Down
5 changes: 3 additions & 2 deletions docs/hooks/lifecycle.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,10 @@ Each result is what `predict` returns for that request, so Router-level predict
request here too: every request gets its own `PredictContext`, `run_id` and `elapsed_ms`.

```
Router.predict_batch(requests, batch_size=...)
Router.predict_batch(requests, batch_size=..., hooks=...)
│
├─ route_batch(requests) ──► on_route, once per request (no checkpoint loaded yet)
├─ active = installed hooks + per-call hooks (installed first; None and [] add nothing)
├─ route_batch(requests, hooks=per-call) ──► on_route, once per request (no checkpoint loaded yet)
│
└─ for each checkpoint, in order of first appearance:
│
Expand Down
3 changes: 3 additions & 0 deletions laya-ts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,9 @@ const results = await agent.predictBatch(states, questions, { batchSize: 32 });
// in as few forward passes as possible. Results keep input order and carry `routing`.
const decisions = router.routeBatch(requests); // validate + route, nothing loads
const routed = await router.predictBatch(requests, 32); // == router.predictMany(...)
// Per-call hooks are composed like predict: installed hooks first, then this list.
// null and [] add nothing. The list runs once per request, including onRoute.
const traced = await router.predictBatch(requests, null, { hooks: [tracer] });

// requests routed to the same checkpoint still split into separate batches when their
// question schemas differ (order-sensitively), when per-request start hooks set
Expand Down
2 changes: 1 addition & 1 deletion laya-ts/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export type {
export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle, loadWebBundle, PINNED_REVISIONS, resolveRevision } from "./providers.js";
export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js";
export { Router, normaliseName, DEFAULT_MODELS, digestsFromEnv, _digestsFromEnv } from "./router.js";
export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest, RouterOptions, RouteOptions } from "./router.js";
export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest, PredictBatchOptions, RouterOptions, RouteOptions } from "./router.js";
export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K } from "./shortlist.js";
export type { EmbedFn, ShortlistMeta } from "./shortlist.js";
export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js";
Expand Down
23 changes: 19 additions & 4 deletions laya-ts/src/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,15 @@ export interface RouteOptions {
hooksRaise?: boolean;
}

/**
* Per-call options for `Router.predictBatch` / `predictMany` / `routeBatch`.
* `hooks` is composed the same way as on `predict`: installed hooks first, then this list.
* `null`, `undefined` and `[]` add nothing.
*/
export interface PredictBatchOptions {
hooks?: HookArg;
}

/** One item of a Router.predictBatch/routeBatch batch: state, questions, route overrides. */
export interface BatchRequest {
state: unknown;
Expand Down Expand Up @@ -682,7 +691,7 @@ export class Router extends HookRegistry {
* validated before anything loads, so a malformed batch fails fast. Entries keep input
* order; routing is deterministic, so repeated batches route identically.
*/
routeBatch(requests: BatchRequest[]): RouteDecision[] {
routeBatch(requests: BatchRequest[], opts: PredictBatchOptions = {}): RouteDecision[] {
if (!Array.isArray(requests)) {
throw new TypeError("requests must be an array of request objects");
}
Expand Down Expand Up @@ -711,6 +720,7 @@ export class Router extends HookRegistry {
task: request.task ?? null,
lang: request.lang ?? null,
langGuess: request.langGuess ?? request.lang_guess ?? null,
hooks: opts.hooks,
}),
);
}
Expand All @@ -731,12 +741,16 @@ export class Router extends HookRegistry {
* Router-level predict hooks run per request: each request gets its own PredictContext
* carrying `decision`; `onPredictStart` may rewrite a request or `ctx.skip()` it, and
* `onPredictEnd` runs once per started request even when the batch fails.
* `opts.hooks` are per-call hooks, appended after any installed on the Router, the same
* composition `predict` uses. `null` and `[]` add nothing. They apply to `onRoute` as well
* as to each request's predict lifecycle.
*/
async predictBatch(
requests: BatchRequest[],
batchSize: number | null = null,
opts: PredictBatchOptions = {},
): Promise<RoutedResult[]> {
const decisions = this.routeBatch(requests);
const decisions = this.routeBatch(requests, opts);
if (decisions.length === 0) return [];

const groups = new Map<string, number[]>();
Expand All @@ -748,7 +762,7 @@ export class Router extends HookRegistry {
}

const results: (RoutedResult | null)[] = new Array(requests.length).fill(null);
const active = composeHooks(this.hooks);
const active = composeHooks(this.hooks, opts.hooks);
const raiseErrors = this.hooksRaise;

for (const [modelName, indices] of groups) {
Expand Down Expand Up @@ -924,8 +938,9 @@ export class Router extends HookRegistry {
async predictMany(
requests: BatchRequest[],
batchSize: number | null = null,
opts: PredictBatchOptions = {},
): Promise<RoutedResult[]> {
return this.predictBatch(requests, batchSize);
return this.predictBatch(requests, batchSize, opts);
}

/**
Expand Down
74 changes: 74 additions & 0 deletions laya-ts/tests/router-batch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,80 @@ describe("Router.routeBatch / predictBatch", () => {
]);
});

it("accepts per-call hooks in predict order, and treats null and [] as no-ops", async () => {
const order: string[] = [];
const installed = {
onRoute(ctx: { states: unknown[] }) {
order.push(`installed:route:${ctx.states[0]}`);
},
onPredictStart(ctx: { states: unknown[] }) {
order.push(`installed:start:${ctx.states[0]}`);
},
onPredictEnd(ctx: { states: unknown[] }) {
order.push(`installed:end:${ctx.states[0]}`);
},
};
const perCall = {
onRoute(ctx: { states: unknown[] }) {
order.push(`percall:route:${ctx.states[0]}`);
},
onPredictStart(ctx: { states: unknown[] }) {
order.push(`percall:start:${ctx.states[0]}`);
},
onPredictEnd(ctx: { states: unknown[] }) {
order.push(`percall:end:${ctx.states[0]}`);
},
};
const { router } = makeRouter({ hooks: [installed] });
const items = [req("one", { model: "english" }), req("two", { model: "english" })];
const results = await router.predictBatch(items, null, { hooks: [perCall] });
expect(results.map((r) => r.answers.seen)).toEqual(["one", "two"]);
// routeBatch runs before any predict hook. End hooks follow start order on this runtime.
expect(order).toEqual([
"installed:route:one",
"percall:route:one",
"installed:route:two",
"percall:route:two",
"installed:start:one",
"percall:start:one",
"installed:start:two",
"percall:start:two",
"installed:end:one",
"percall:end:one",
"installed:end:two",
"percall:end:two",
]);

order.length = 0;
await router.predictBatch(items, null, { hooks: null });
const noneOrder = [...order];
order.length = 0;
await router.predictBatch(items, null, { hooks: [] });
const emptyOrder = [...order];
order.length = 0;
await router.predictBatch(items);
expect(noneOrder).toEqual(emptyOrder);
expect(emptyOrder).toEqual(order);
expect(noneOrder).toEqual([
"installed:route:one",
"installed:route:two",
"installed:start:one",
"installed:start:two",
"installed:end:one",
"installed:end:two",
]);

const { router: bare } = makeRouter();
expect((await bare.predictBatch(items, null, { hooks: null })).map((r) => r.answers.seen)).toEqual([
"one",
"two",
]);
expect((await bare.predictBatch(items, null, { hooks: [] })).map((r) => r.answers.seen)).toEqual([
"one",
"two",
]);
});

it("runs router hooks per request and short-circuits skipped requests", async () => {
const events: string[] = [];
const { router, calls } = makeRouter({
Expand Down
12 changes: 11 additions & 1 deletion laya/mcp/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,9 @@ def predict(
head_max_len: Optional[int] = None,
min_confidence: Optional[float] = None,
) -> Dict[str, Any]:
if any(value is not None for value in (hooks, on_predict_start, on_predict_end, hooks_raise, hooks_timeout)):
if hooks_timeout is not None:
raise RemoteError("unsupported_remote", "hooks_timeout belongs to the server's hooks")
if any(value is not None for value in (hooks, on_predict_start, on_predict_end, hooks_raise)):
raise RemoteError("unsupported_remote",
"hooks are callables and cannot be sent to laya-serve; install them on the server")
body: Dict[str, Any] = {"state": state, "questions": questions}
Expand All @@ -196,6 +198,11 @@ def predict_batch(
hooks_timeout: Optional[float] = None,
min_confidence: Optional[float] = None,
sort_by_length: bool = False,
*,
hooks=None,
on_predict_start=None,
on_predict_end=None,
hooks_raise: Optional[bool] = None,
) -> List[Dict[str, Any]]:
"""One ``/v1/systemone`` call per request, answers in input order.

Expand All @@ -208,6 +215,9 @@ def predict_batch(
results: List[Dict[str, Any]] = []
if hooks_timeout is not None:
raise RemoteError("unsupported_remote", "hooks_timeout belongs to the server's hooks")
if any(value is not None for value in (hooks, on_predict_start, on_predict_end, hooks_raise)):
raise RemoteError("unsupported_remote",
"hooks are callables and cannot be sent to laya-serve; install them on the server")
for item in requests:
if not isinstance(item, dict):
raise TypeError("predict_batch requests must be dicts, got %s" % type(item).__name__)
Expand Down
23 changes: 20 additions & 3 deletions laya/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -1167,6 +1167,9 @@ def route_batch(
self,
requests: Sequence[Dict[str, Any]],
hooks_timeout: Optional[float] = None,
*,
hooks=None,
hooks_raise: Optional[bool] = None,
) -> List[RouteDecision]:
"""Route a heterogeneous request batch without loading any checkpoints.

Expand All @@ -1182,6 +1185,8 @@ def route_batch(
``questions``.
hooks_timeout: Override the Router's ``hooks_timeout`` for this call, applied to
every request's ``on_route`` dispatch, as on :meth:`route`.
hooks (HookArg): Per-call hook or sequence of hooks for this call.
hooks_raise: Override the Router's ``hooks_raise`` policy for this call.
"""
if not isinstance(requests, SequenceABC) or isinstance(requests, (str, bytes)):
raise TypeError("requests must be a sequence of request dictionaries")
Expand Down Expand Up @@ -1210,6 +1215,8 @@ def route_batch(
task=request.get("task"),
lang=request.get("lang"),
lang_guess=request.get("lang_guess"),
hooks=hooks,
hooks_raise=hooks_raise,
hooks_timeout=hooks_timeout,
)
)
Expand All @@ -1223,6 +1230,11 @@ def predict_batch(
hooks_timeout: Optional[float] = None,
min_confidence: Optional[float] = None,
sort_by_length: bool = False,
*,
hooks=None,
on_predict_start=None,
on_predict_end=None,
hooks_raise: Optional[bool] = None,
) -> List[Dict[str, Any]]:
"""Route and execute a heterogeneous request batch with minimal model churn.

Expand Down Expand Up @@ -1256,16 +1268,21 @@ def predict_batch(
overrides.
batch_size: Optional maximum number of states per Agent forward-pass batch.
hooks_timeout: Override the Router's ``hooks_timeout`` for this call.
min_confidence: Optional float or per-bucket mapping for confidence gating.
sort_by_length: Forwarded to every ``Agent.predict_batch`` call, so each question
group pads to a shorter maximum; see ``Agent.predict_batch``. Results retain the
input order either way. Silently dropped for an attached agent whose
``predict_batch`` predates the knob (#294).
hooks (HookArg): Per-call hook or sequence of hooks for this call.
on_predict_start (PredictHookArg): Plain callable or sequence of callables for start events.
on_predict_end (PredictHookArg): Plain callable or sequence of callables for end events.
hooks_raise: Override the Router's ``hooks_raise`` policy for this call.

Returns:
One normal Router prediction result per request, in the same order as the input.
"""
mc = check_min_confidence(min_confidence) if min_confidence is not None else None
decisions = self.route_batch(requests, hooks_timeout=hooks_timeout)
decisions = self.route_batch(requests, hooks=hooks, hooks_raise=hooks_raise, hooks_timeout=hooks_timeout)
if not decisions:
return []

Expand All @@ -1288,8 +1305,8 @@ def predict_batch(
# list alone silently dropped every process-wide default from the batched path while
# keeping them on `predict`, so a default audit or metrics hook saw no Router-level event
# for a request that arrived through `predict_batch`.
active = compose_hooks(self.hooks)
raise_errors = self.hooks_raise
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)

for model_name, indices in groups.items():
Expand Down
Loading
Loading