Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions laya/mcp/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,14 +306,24 @@ def laya_predict_batch_tool(requests: list, batch_size: int = 0) -> str:
+ batch_item_key_doc(omit=("max_len", "head_max_len")) + ". Use this to "
"inspect or aggregate the routing of a workload before paying model-load cost. "
"Returns one {model, repo, reason} decision per request in input order, plus "
"model_counts. "
"model_counts. hooks_timeout: positive number or unset -- override the "
"Router's own hook deadline for this sweep's on_route dispatch, so a slow "
"or hung operator hook cannot stall a routing pass the caller only wants "
"the checkpoint labels for. "
+ _GUARDRAILS
),
)
def laya_route_batch_tool(requests: list) -> str:
def laya_route_batch_tool(requests: list, hooks_timeout: float = 0) -> str:
"""Route many requests to checkpoints without a forward pass."""
router = _router_or_error()
return _wrap(laya_route_batch, requests=requests, router=router)
# hooks_timeout=0 means "unset": MCP clients send defaults eagerly, and
# None is what Router.route_batch takes as "use my own timeout".
return _wrap(
laya_route_batch,
requests=requests,
hooks_timeout=hooks_timeout or None,
router=router,
)


@server.tool(
Expand Down
30 changes: 28 additions & 2 deletions laya/mcp/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from typing import Any, Callable, Protocol, Sequence

from ..confidence import check_min_confidence
from ..hooks import validate_timeout
from ..presets import state_field
from .device import agent_device, device_report, router_agent

Expand Down Expand Up @@ -908,6 +909,26 @@ def _validate_batch_size(batch_size: Any) -> int | None:
return batch_size


def _validate_hooks_timeout(value: Any) -> float | None:
# Mirrors laya_predict_batch's forwarding shape: unset stays unset so a
# Router with its own default is not shadowed by 0, and core's
# ``validate_timeout`` decides what counts as a real deadline. Wrapping
# its ValueError as a ToolError keeps a bad arg from surfacing as
# ``internal_error: ValueError`` at the MCP boundary. Bools are refused
# here the same way _validate_batch_size refuses them -- core's
# float(True) == 1.0 would silently turn "True" into a one-second
# deadline, which is the kind of wrong-that-needs-to-shout.
if value is None:
return None
if isinstance(value, bool):
raise ToolError("invalid_hooks_timeout",
"hooks_timeout must be a positive number or None, got %r" % (value,))
try:
return validate_timeout(value)
except (TypeError, ValueError) as exc:
raise ToolError("invalid_hooks_timeout", str(exc)) from exc


def laya_predict_batch(
requests: Any,
batch_size: Any = None,
Expand Down Expand Up @@ -975,19 +996,24 @@ def laya_predict_batch(
}


def laya_route_batch(requests: Any, *, router: Any = None) -> dict:
def laya_route_batch(requests: Any, hooks_timeout: Any = None, *, router: Any = None) -> dict:
"""Routing decisions for many requests: no forward pass, no checkpoint loads.

The batch form of ``laya_route``, mirroring ``Router.route_batch``: it
reports which checkpoint each request *would* answer from so clients can
inspect or aggregate a workload's routing before paying any load cost.
``hooks_timeout`` overrides the Router's own value for this call's
``on_route`` dispatch, exactly as it does for ``laya_predict_batch``: an
operator-installed hook that hangs should not stall a whole routing sweep.
"""
items = validate_batch_requests(requests)
timeout = _validate_hooks_timeout(hooks_timeout)
if router is None:
raise ToolError("models_not_ready", "Router is not loaded")
if not hasattr(router, "route_batch"):
raise ToolError("internal_error", "router has no route_batch() method")
decisions = router.route_batch(items)
decisions = router.route_batch(items, hooks_timeout=timeout) if timeout is not None \
else router.route_batch(items)
if not isinstance(decisions, list) or len(decisions) != len(items):
raise ToolError(
"internal_error",
Expand Down
64 changes: 61 additions & 3 deletions tests/test_mcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -846,8 +846,8 @@ def predict_batch(self, requests, batch_size=None):
self.predict_batch_calls.append((list(requests), batch_size))
return [self._answer_for(request) for request in requests]

def route_batch(self, requests):
self.route_batch_calls.append(list(requests))
def route_batch(self, requests, hooks_timeout=None):
self.route_batch_calls.append((list(requests), hooks_timeout))
return [{"model": request.get("model") or "english", "repo": "fake/laya",
"reason": "batch route"} for request in requests]

Expand All @@ -858,10 +858,24 @@ class ShortRouter:
def predict_batch(self, requests, batch_size=None):
return []

def route_batch(self, requests):
def route_batch(self, requests, hooks_timeout=None):
return []


class StrictRouteRouter:
"""A Router stub that predates the hooks_timeout kwarg on route_batch.

laya_route_batch must send `route_batch(items)` when the caller did not
ask for a hook deadline -- sending `route_batch(items, hooks_timeout=None)`
would break every caller that attached a pre-#577 Router (or a stub like
this one). This is the witness for M2 in the PR's mutation harness.
"""

def route_batch(self, requests):
return [{"model": "english", "repo": "fake/laya",
"reason": "strict route"} for _ in requests]


BATCH_REQUESTS = [
{"state": {"body": "refund please"}, "questions": QUESTIONS},
{"state": {"body": "please refund invoice 2"}, "questions": QUESTIONS,
Expand Down Expand Up @@ -1023,6 +1037,50 @@ def test_batch_route():
ok("batch/route_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))
# Route-only never predicts.
ok("batch/route_no_forward", router.predict_batch_calls == [])
# Unset must not shadow the Router's own timeout: the tool layer only
# forwards when the caller actually asked for one.
ok("batch/route_hooks_timeout_default_none",
router.route_batch_calls[0][1] is None, repr(router.route_batch_calls[0]))
# The unset case must ALSO work against a Router that has never heard of
# hooks_timeout: sending `route_batch(items, hooks_timeout=None)` would
# break every caller with an older or hand-rolled router. Wrapped so the
# TypeError shows as a NAMED FAIL rather than crashing the whole suite.
try:
strict_out = laya_route_batch(BATCH_REQUESTS, router=StrictRouteRouter())
strict_ok = (len(strict_out["decisions"]) == 3
and strict_out["model_counts"] == {"english": 3})
detail = repr(strict_out)
except TypeError as exc:
strict_ok = False
detail = "TypeError: %s" % exc
ok("batch/route_strict_router_no_kwarg", strict_ok, detail)
# hooks_timeout is forwarded verbatim (post-validate_timeout) to
# Router.route_batch's per-call override of the operator-installed hook
# deadline. Same shape as laya_predict_batch (see #766).
router = BatchRouter()
laya_route_batch(BATCH_REQUESTS, hooks_timeout=2.5, router=router)
_, timeout = router.route_batch_calls[0]
ok("batch/route_hooks_timeout_forwarded", timeout == 2.5, repr(timeout))
# An int is a valid timeout -- validate_timeout coerces to float.
router = BatchRouter()
laya_route_batch(BATCH_REQUESTS, hooks_timeout=1, router=router)
_, timeout = router.route_batch_calls[0]
ok("batch/route_hooks_timeout_int_coerced", timeout == 1.0 and isinstance(timeout, float),
repr(timeout))
# Bad values become invalid_hooks_timeout, not internal_error: the
# boundary check exists so a caller gets a real code back over MCP.
for bad in (0, -1, -0.5, "soon", True, [1.0]):
expect_tool_error("batch/route_bad_hooks_timeout_%r" % (bad,),
lambda b=bad: laya_route_batch(
BATCH_REQUESTS, hooks_timeout=b, router=BatchRouter()),
"invalid_hooks_timeout")
# Validation happens before the router is touched: one bad timeout, no
# partial dispatch.
router = BatchRouter()
expect_tool_error("batch/route_bad_hooks_timeout_zero_router_untouched",
lambda: laya_route_batch(BATCH_REQUESTS, hooks_timeout=0, router=router),
"invalid_hooks_timeout")
ok("batch/route_no_dispatch_on_bad_timeout", router.route_batch_calls == [])
expect_tool_error("batch/route_count_mismatch",
lambda: laya_route_batch(BATCH_REQUESTS, router=ShortRouter()),
"internal_error")
Expand Down
Loading