Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs/hooks/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,9 @@ router.predict(state, questions, model=None, task=None, lang=None, lang_guess=No
hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
hooks_timeout=None, max_len=None, head_max_len=None)

router.predict_batch(requests, batch_size=None, hooks_timeout=None, min_confidence=None,
sort_by_length=False, hooks=None)

router.system_one(...) # alias of predict
router.load(name) # builds on first use; fires on_load
router.preload(names=None) # builds several; fires on_load per build
Expand All @@ -304,7 +307,7 @@ router.attach(name, agent) # registers an existing agent; does not fire on_load
router.loaded # list of resident checkpoint names
```

- Per-call `hooks=` on `route` and `predict` apply to the whole call, including `on_route`.
- Per-call `hooks=` on `route`, `predict` and `predict_batch` apply to the whole call, including `on_route`. On `predict_batch` the list is composed the same way as on `predict` (installed hooks first, then the per-call list; `None` and `[]` add nothing) and runs once per request.
- `route()` is public: calling it dispatches `on_route` with the installed hooks plus any
per-call `hooks`.

Expand Down
5 changes: 3 additions & 2 deletions docs/hooks/lifecycle.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,10 @@ Each result is what `predict` returns for that request, so Router-level predict
request here too: every request gets its own `PredictContext`, `run_id` and `elapsed_ms`.

```
Router.predict_batch(requests, batch_size=...)
Router.predict_batch(requests, batch_size=..., hooks=...)
│
├─ route_batch(requests) ──► on_route, once per request (no checkpoint loaded yet)
├─ active = installed hooks + per-call hooks (installed first; None and [] add nothing)
├─ route_batch(requests, hooks=per-call) ──► on_route, once per request (no checkpoint loaded yet)
│
└─ for each checkpoint, in order of first appearance:
│
Expand Down
3 changes: 3 additions & 0 deletions laya-ts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,9 @@ const results = await agent.predictBatch(states, questions, { batchSize: 32 });
// in as few forward passes as possible. Results keep input order and carry `routing`.
const decisions = router.routeBatch(requests); // validate + route, nothing loads
const routed = await router.predictBatch(requests, 32); // == router.predictMany(...)
// Per-call hooks are composed like predict: installed hooks first, then this list.
// null and [] add nothing. The list runs once per request, including onRoute.
const traced = await router.predictBatch(requests, null, { hooks: [tracer] });

// requests routed to the same checkpoint still split into separate batches when their
// question schemas differ (order-sensitively), when per-request start hooks set
Expand Down
2 changes: 1 addition & 1 deletion laya-ts/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export type {
export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle, loadWebBundle, PINNED_REVISIONS, resolveRevision } from "./providers.js";
export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js";
export { Router, normaliseName, DEFAULT_MODELS, digestsFromEnv, _digestsFromEnv } from "./router.js";
export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest, RouterOptions, RouteOptions } from "./router.js";
export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest, PredictBatchOptions, RouterOptions, RouteOptions } from "./router.js";
export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K } from "./shortlist.js";
export type { EmbedFn, ShortlistMeta } from "./shortlist.js";
export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js";
Expand Down
23 changes: 19 additions & 4 deletions laya-ts/src/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,15 @@ export interface RouteOptions {
hooksRaise?: boolean;
}

/**
* Per-call options for `Router.predictBatch` / `predictMany` / `routeBatch`.
* `hooks` is composed the same way as on `predict`: installed hooks first, then this list.
* `null`, `undefined` and `[]` add nothing.
*/
export interface PredictBatchOptions {
hooks?: HookArg;
}

/** One item of a Router.predictBatch/routeBatch batch: state, questions, route overrides. */
export interface BatchRequest {
state: unknown;
Expand Down Expand Up @@ -682,7 +691,7 @@ export class Router extends HookRegistry {
* validated before anything loads, so a malformed batch fails fast. Entries keep input
* order; routing is deterministic, so repeated batches route identically.
*/
routeBatch(requests: BatchRequest[]): RouteDecision[] {
routeBatch(requests: BatchRequest[], opts: PredictBatchOptions = {}): RouteDecision[] {
if (!Array.isArray(requests)) {
throw new TypeError("requests must be an array of request objects");
}
Expand Down Expand Up @@ -711,6 +720,7 @@ export class Router extends HookRegistry {
task: request.task ?? null,
lang: request.lang ?? null,
langGuess: request.langGuess ?? request.lang_guess ?? null,
hooks: opts.hooks,
}),
);
}
Expand All @@ -731,12 +741,16 @@ export class Router extends HookRegistry {
* Router-level predict hooks run per request: each request gets its own PredictContext
* carrying `decision`; `onPredictStart` may rewrite a request or `ctx.skip()` it, and
* `onPredictEnd` runs once per started request even when the batch fails.
* `opts.hooks` are per-call hooks, appended after any installed on the Router, the same
* composition `predict` uses. `null` and `[]` add nothing. They apply to `onRoute` as well
* as to each request's predict lifecycle.
*/
async predictBatch(
requests: BatchRequest[],
batchSize: number | null = null,
opts: PredictBatchOptions = {},
): Promise<RoutedResult[]> {
const decisions = this.routeBatch(requests);
const decisions = this.routeBatch(requests, opts);
if (decisions.length === 0) return [];

const groups = new Map<string, number[]>();
Expand All @@ -748,7 +762,7 @@ export class Router extends HookRegistry {
}

const results: (RoutedResult | null)[] = new Array(requests.length).fill(null);
const active = composeHooks(this.hooks);
const active = composeHooks(this.hooks, opts.hooks);
const raiseErrors = this.hooksRaise;

for (const [modelName, indices] of groups) {
Expand Down Expand Up @@ -924,8 +938,9 @@ export class Router extends HookRegistry {
async predictMany(
requests: BatchRequest[],
batchSize: number | null = null,
opts: PredictBatchOptions = {},
): Promise<RoutedResult[]> {
return this.predictBatch(requests, batchSize);
return this.predictBatch(requests, batchSize, opts);
}

/**
Expand Down
74 changes: 74 additions & 0 deletions laya-ts/tests/router-batch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,80 @@ describe("Router.routeBatch / predictBatch", () => {
]);
});

it("accepts per-call hooks in predict order, and treats null and [] as no-ops", async () => {
const order: string[] = [];
const installed = {
onRoute(ctx: { states: unknown[] }) {
order.push(`installed:route:${ctx.states[0]}`);
},
onPredictStart(ctx: { states: unknown[] }) {
order.push(`installed:start:${ctx.states[0]}`);
},
onPredictEnd(ctx: { states: unknown[] }) {
order.push(`installed:end:${ctx.states[0]}`);
},
};
const perCall = {
onRoute(ctx: { states: unknown[] }) {
order.push(`percall:route:${ctx.states[0]}`);
},
onPredictStart(ctx: { states: unknown[] }) {
order.push(`percall:start:${ctx.states[0]}`);
},
onPredictEnd(ctx: { states: unknown[] }) {
order.push(`percall:end:${ctx.states[0]}`);
},
};
const { router } = makeRouter({ hooks: [installed] });
const items = [req("one", { model: "english" }), req("two", { model: "english" })];
const results = await router.predictBatch(items, null, { hooks: [perCall] });
expect(results.map((r) => r.answers.seen)).toEqual(["one", "two"]);
// routeBatch runs before any predict hook. End hooks follow start order on this runtime.
expect(order).toEqual([
"installed:route:one",
"percall:route:one",
"installed:route:two",
"percall:route:two",
"installed:start:one",
"percall:start:one",
"installed:start:two",
"percall:start:two",
"installed:end:one",
"percall:end:one",
"installed:end:two",
"percall:end:two",
]);

order.length = 0;
await router.predictBatch(items, null, { hooks: null });
const noneOrder = [...order];
order.length = 0;
await router.predictBatch(items, null, { hooks: [] });
const emptyOrder = [...order];
order.length = 0;
await router.predictBatch(items);
expect(noneOrder).toEqual(emptyOrder);
expect(emptyOrder).toEqual(order);
expect(noneOrder).toEqual([
"installed:route:one",
"installed:route:two",
"installed:start:one",
"installed:start:two",
"installed:end:one",
"installed:end:two",
]);

const { router: bare } = makeRouter();
expect((await bare.predictBatch(items, null, { hooks: null })).map((r) => r.answers.seen)).toEqual([
"one",
"two",
]);
expect((await bare.predictBatch(items, null, { hooks: [] })).map((r) => r.answers.seen)).toEqual([
"one",
"two",
]);
});

it("runs router hooks per request and short-circuits skipped requests", async () => {
const events: string[] = [];
const { router, calls } = makeRouter({
Expand Down
9 changes: 8 additions & 1 deletion laya/mcp/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import urllib.parse
from typing import Any, Dict, List, Optional, Sequence, Union

from ..hooks import HookArg
from ..router import Router
from ..serve import BODY_CONTROLS, BODY_REFUSALS

Expand Down Expand Up @@ -196,16 +197,22 @@ def predict_batch(
hooks_timeout: Optional[float] = None,
min_confidence: Optional[float] = None,
sort_by_length: bool = False,
hooks: HookArg = None,
) -> List[Dict[str, Any]]:
"""One ``/v1/systemone`` call per request, answers in input order.

``laya-serve``'s ``/v1/systemone/batch`` shares one ``questions`` object across every state,
while a Router batch lets each request carry its own questions and controls, so the general
form is the per-request call. ``batch_size`` and ``sort_by_length`` shape forward passes and
have no meaning here; they are accepted and ignored so callers written against ``Router``
keep working. The server applies its own batching inside each call.
keep working. The server applies its own batching inside each call. ``hooks`` is accepted
so the signature matches ``Router.predict_batch`` and refused: a hook is a callable and
cannot be sent to ``laya-serve``.
"""
results: List[Dict[str, Any]] = []
if hooks is not None:
raise RemoteError("unsupported_remote",
"hooks are callables and cannot be sent to laya-serve; install them on the server")
if hooks_timeout is not None:
raise RemoteError("unsupported_remote", "hooks_timeout belongs to the server's hooks")
for item in requests:
Expand Down
35 changes: 24 additions & 11 deletions laya/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@

from .confidence import apply_confidence_gate, check_min_confidence
from .hooks import (
HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,
validate_timeout,
HookArg, HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch,
normalise_hooks, validate_timeout,
)
from .hooks import _SKIP_DEFAULTS
from .lang import analyse
Expand Down Expand Up @@ -1167,6 +1167,7 @@ def route_batch(
self,
requests: Sequence[Dict[str, Any]],
hooks_timeout: Optional[float] = None,
hooks: HookArg = None,
) -> List[RouteDecision]:
"""Route a heterogeneous request batch without loading any checkpoints.

Expand All @@ -1182,6 +1183,9 @@ def route_batch(
``questions``.
hooks_timeout: Override the Router's ``hooks_timeout`` for this call, applied to
every request's ``on_route`` dispatch, as on :meth:`route`.
hooks (HookArg): Per-call hooks, appended after any installed on the Router and
applied to every request's ``on_route`` dispatch, as on :meth:`route`. ``None``
and ``[]`` add nothing.
"""
if not isinstance(requests, SequenceABC) or isinstance(requests, (str, bytes)):
raise TypeError("requests must be a sequence of request dictionaries")
Expand Down Expand Up @@ -1210,6 +1214,7 @@ def route_batch(
task=request.get("task"),
lang=request.get("lang"),
lang_guess=request.get("lang_guess"),
hooks=hooks,
hooks_timeout=hooks_timeout,
)
)
Expand All @@ -1223,6 +1228,7 @@ def predict_batch(
hooks_timeout: Optional[float] = None,
min_confidence: Optional[float] = None,
sort_by_length: bool = False,
hooks: HookArg = None,
) -> List[Dict[str, Any]]:
"""Route and execute a heterogeneous request batch with minimal model churn.

Expand Down Expand Up @@ -1260,12 +1266,21 @@ def predict_batch(
group pads to a shorter maximum; see ``Agent.predict_batch``. Results retain the
input order either way. Silently dropped for an attached agent whose
``predict_batch`` predates the knob (#294).
hooks (HookArg): Per-call hooks, composed the same way as on :meth:`predict`:
installed hooks first, then this list. ``None`` and ``[]`` add nothing. The
list runs for every request, including ``on_route`` inside :meth:`route_batch`.

Returns:
One normal Router prediction result per request, in the same order as the input.
"""
mc = check_min_confidence(min_confidence) if min_confidence is not None else None
decisions = self.route_batch(requests, hooks_timeout=hooks_timeout)
# Composed before routing, as `predict` does, so a bad per-call hook raises before any
# checkpoint loads and the same list reaches `on_route` and the predict lifecycle.
# `None` and `[]` normalise to nothing, so they leave the installed hooks alone.
active = compose_hooks(self.hooks, hooks)
raise_errors = self.hooks_raise
timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
decisions = self.route_batch(requests, hooks_timeout=hooks_timeout, hooks=hooks)
if not decisions:
return []

Expand All @@ -1283,14 +1298,12 @@ def predict_batch(
# `predict` returns as it is.
results: List[Any] = [None] * len(requests)
answered = 0
# `compose_hooks`, not `list(self.hooks)`: this is the composition `predict` uses at its
# own dispatch site, and it is what merges in `set_default_hooks`. Reading the instance
# list alone silently dropped every process-wide default from the batched path while
# keeping them on `predict`, so a default audit or metrics hook saw no Router-level event
# for a request that arrived through `predict_batch`.
active = compose_hooks(self.hooks)
raise_errors = self.hooks_raise
timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
# `active` is the composition `predict` uses at its own dispatch site (`compose_hooks`,
# not `list(self.hooks)`), including per-call `hooks=`. That is what merges in
# `set_default_hooks`. Reading the instance list alone silently dropped every
# process-wide default from the batched path while keeping them on `predict`, so a
# default audit or metrics hook saw no Router-level event for a request that arrived
# through `predict_batch`.

for model_name, indices in groups.items():
agent = self.load(model_name)
Expand Down
7 changes: 4 additions & 3 deletions laya/serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,9 +145,10 @@
# against ``inspect.signature(Router.predict_batch)`` (call-level) and against the MCP tool's own
# item key list (per-request), so a control added to either side has to be placed on one side of
# this split -- forwarded, refused, or moved to the item tuple -- before the suite goes green.
# ``hooks_timeout`` is refused on the batch path the same way it is on the single path: it belongs
# to ``predict_batch``'s call-level args but governs how the *deployment's* hooks execute, so a
# caller cannot be allowed to shorten or lengthen that deadline from an HTTP body.
# ``hooks`` and ``hooks_timeout`` are refused on the batch path the same way they are on the single
# path: they belong to ``predict_batch``'s call-level args, but a hook is a callable that runs in
# the server process and ``hooks_timeout`` governs how the *deployment's* hooks execute, so a
# caller cannot send either from an HTTP body.
BATCH_BODY_CALL_CONTROLS = ("batch_size", "min_confidence", "sort_by_length")
BATCH_BODY_ITEM_CONTROLS = ("max_len", "head_max_len", "task", "lang", "lang_guess")

Expand Down
1 change: 1 addition & 0 deletions tests/test_hooks_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ def check_param(name, fn, param, default, kind=None):
# the request keys `max_len` / `head_max_len` are read into each request's PredictContext.
check_param("Router.predict_batch", Router.predict_batch, "batch_size", None)
check_param("Router.predict_batch", Router.predict_batch, "hooks_timeout", None)
check_param("Router.predict_batch", Router.predict_batch, "hooks", None)
for param in ("max_len", "head_max_len"):
check("Router.predict_batch/%s is per-request, not a call argument" % param,
param in sig(Router.predict_batch), False)
Expand Down
2 changes: 2 additions & 0 deletions tests/test_mcp_remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,8 @@ def test_remote_batch_refuses_hook_timeouts_and_item_hooks(serve):
items = [{"state": "x", "questions": _questions()}]
with pytest.raises(RemoteError, match="hooks_timeout"):
router.predict_batch(items, hooks_timeout=1)
with pytest.raises(RemoteError, match="callables"):
router.predict_batch(items, hooks=[])
items[0]["hooks"] = [lambda ctx: None]
with pytest.raises(RemoteError, match="per-request hooks"):
router.predict_batch(items)
Expand Down
Loading