From 9189863c3d4095f81a9298fcc1d8ac49fce3482a Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:10:10 +0545 Subject: [PATCH] feat(mcp): expose core's min_confidence abstention control on the decision tools min_confidence (#361) is a validated keyword of Agent.predict, Router.predict/ predict_batch and laya.decide on main, but the MCP layer never forwarded it: a client could pin a checkpoint and size its token budget over the wire yet could not ask Laya to abstain on an unsure answer. laya_predict, laya_shortlist, laya_preset and laya_decide now take it and forward it to core, validated in the tool layer via laya.confidence .check_min_confidence so a bad value is an invalid_min_confidence ToolError rather than a forward pass that never runs. Only set values are forwarded, so a no-abstention call stays byte-identical to one before the keyword existed. On laya_decide a field below the threshold comes back null in values while its confidence survives -- parity with laya.decide(min_confidence=...). --- laya/mcp/server.py | 22 ++++++-- laya/mcp/tools.py | 60 +++++++++++++++++--- tests/test_mcp.py | 138 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 12 deletions(-) diff --git a/laya/mcp/server.py b/laya/mcp/server.py index 9e3342e4e..9f2add292 100644 --- a/laya/mcp/server.py +++ b/laya/mcp/server.py @@ -100,8 +100,11 @@ "checkpoint and selects that checkpoint's per-language calibration. " "max_len / head_max_len: positive integers overriding the answering token budget for this call " "only -- head_max_len is the option-and-instructions budget, so raise it when a choice question " - "has many options and the answers look like the labels blur together. Leave any of them unset to " - "keep the checkpoint's own default." + "has many options and the answers look like the labels blur together. " + "min_confidence: a number in [0, 1] -- an answer whose calibrated confidence falls below it is " + "flagged 'low_confidence' (and, on laya_decide, its value comes back null), so a caller that " + "must not act on a guess can set it. Leave any of them unset to keep the checkpoint's own " + "default." ) @@ -248,6 +251,7 @@ def laya_predict_tool( lang: str | None = None, max_len: int | None = None, head_max_len: int | None = None, + min_confidence: float | None = None, ) -> str: """Answer typed questions (choice/score/noul) over any state in one forward pass.""" router = _router_or_error() @@ -260,6 +264,7 @@ def laya_predict_tool( lang=lang, max_len=max_len, head_max_len=head_max_len, + min_confidence=min_confidence, router=router, ) @@ -332,6 +337,7 @@ def laya_shortlist_tool( lang: str | None = None, max_len: int | None = None, head_max_len: int | None = None, + min_confidence: float | None = None, ) -> str: """Shortlist many-option choice questions, then answer.""" # k's default mirrors laya.shortlist.DEFAULT_SHORTLIST_K; it is a literal @@ -348,6 +354,7 @@ def laya_shortlist_tool( lang=lang, max_len=max_len, head_max_len=head_max_len, + min_confidence=min_confidence, router=router, ) @@ -386,6 +393,7 @@ def laya_preset_tool( lang: str | None = None, max_len: int | None = None, head_max_len: int | None = None, + min_confidence: float | None = None, ) -> str: """Run a built-in workflow preset; the tool description carries the names and state fields.""" router = _router_or_error() @@ -397,6 +405,7 @@ def laya_preset_tool( lang=lang, max_len=max_len, head_max_len=head_max_len, + min_confidence=min_confidence, router=router, preset_builder=_preset_builder, ) @@ -412,13 +421,18 @@ def laya_preset_tool( "when the caller already knows the answer shape and wants values projected onto the " "schema (enum member, integer level, boolean) plus per-field confidence, instead of " "an answer map to parse by hand. " + "min_confidence: a number in [0, 1] -- a field whose answer falls below it comes back as " + "null in values (its confidence is still reported), so a caller that must not act on a " + "guess can abstain per field. " + _GUARDRAILS ), ) -def laya_decide_tool(state: dict, schema: dict, model: str = "auto") -> str: +def laya_decide_tool(state: dict, schema: dict, model: str = "auto", + min_confidence: float | None = None) -> str: """Answer a JSON-schema-shaped decision and return the decided values.""" router = _router_or_error() - return _wrap(laya_decide, state=state, schema=schema, model=model, router=router) + return _wrap(laya_decide, state=state, schema=schema, model=model, + min_confidence=min_confidence, router=router) def main() -> None: diff --git a/laya/mcp/tools.py b/laya/mcp/tools.py index ee294383f..cd519de95 100644 --- a/laya/mcp/tools.py +++ b/laya/mcp/tools.py @@ -5,6 +5,7 @@ import time from typing import Any, Callable, Protocol, Sequence +from ..confidence import check_min_confidence from ..presets import state_field from .device import agent_device, device_report, router_agent @@ -293,6 +294,23 @@ def validate_budget(value: Any, name: str) -> int | None: return value +def validate_min_confidence(value: Any) -> float | None: + """The per-call abstention threshold (``min_confidence``), or ``None`` when unset. + + Range, booleans and non-finite values are core's rule, not this layer's, so it defers to + ``laya.confidence.check_min_confidence`` and only wraps its ``ValueError`` as a clean + ``invalid_min_confidence`` the client can read -- the way ``validate_task`` carries core's + task list instead of restating it. ``None`` means "answer everything", which is the flag's + meaning on the CLI and the checkpoint default on every surface. + """ + if value is None: + return None + try: + return check_min_confidence(value) + except (TypeError, ValueError) as exc: + raise ToolError("invalid_min_confidence", str(exc)) from None + + def _overrides(task: Any, lang: Any, max_len: Any, head_max_len: Any) -> dict: """The per-call controls as keyword arguments, with the unset ones left out. @@ -344,6 +362,7 @@ def laya_predict( lang: Any = None, max_len: Any = None, head_max_len: Any = None, + min_confidence: Any = None, router: Any = None, agent: Any = None, ) -> dict: @@ -353,8 +372,9 @@ def laya_predict( ``task``/``lang`` are the router's own overrides (an explicit ``model`` outranks an explicit ``task``, which outranks an explicit ``lang``); ``max_len``/``head_max_len`` override the token - budget the answering checkpoint was configured with. None of them is forwarded unless set, so a - router or agent that predates those keywords keeps working. + budget the answering checkpoint was configured with; ``min_confidence`` flags any answer whose + calibrated confidence falls below it with ``low_confidence: true``. None of them is forwarded + unless set, so a router or agent that predates those keywords keeps working. """ state_d = validate_state(state) questions_d = validate_questions(questions) @@ -366,6 +386,11 @@ def laya_predict( validate_budget(max_len, "max_len"), validate_budget(head_max_len, "head_max_len"), ) + min_conf = validate_min_confidence(min_confidence) + if min_conf is not None: + # Only when set, so a call that abstains over nothing stays byte-identical to the one + # that made no ``min_confidence`` keyword exist. + budget["min_confidence"] = min_conf # `task` picks a checkpoint by saying what the work is, so it means nothing once one is pinned: # `Router._route` checks an explicit `model` first and never reaches the task, and # `Agent.system_one` does not accept the keyword at all. Both would answer as if it were unset, @@ -560,6 +585,7 @@ def laya_shortlist( lang: Any = None, max_len: Any = None, head_max_len: Any = None, + min_confidence: Any = None, router: Any = None, agent: Any = None, embed_fn: Callable[[Sequence[str]], Any] | None = None, @@ -579,7 +605,8 @@ def laya_shortlist( ``head_max_len`` matters more here than anywhere else: shortlisting exists because a large label set shares that budget, and narrowing to ``k`` is only half of the fix. The budget override reaches the answering forward pass; - ``task``/``lang`` reach the route that chose the checkpoint. + ``task``/``lang`` reach the route that chose the checkpoint; ``min_confidence`` + flags a kept-label answer the checkpoint is unsure of. """ # Lazy: keeps numpy/shortlist out of module import for laya.mcp.tools. from laya.shortlist import ( @@ -595,6 +622,11 @@ def laya_shortlist( routing_overrides = _overrides(validate_task(task), validate_lang(lang), None, None) budget = _overrides(None, None, validate_budget(max_len, "max_len"), validate_budget(head_max_len, "head_max_len")) + min_conf = validate_min_confidence(min_confidence) + if min_conf is not None: + # Forwarded to the answering predict the same way the budget is, so a shortlisted field the + # model reads with low confidence comes back flagged rather than as a confident-looking pick. + budget["min_confidence"] = min_conf # Same rule as `laya_predict`: a checkpoint is already pinned, so `task` has nothing left to # decide -- and it is a routing keyword, which the answering `system_one` does not accept. if model_name != AUTO and "task" in routing_overrides: @@ -687,6 +719,7 @@ def laya_preset( lang: Any = None, max_len: Any = None, head_max_len: Any = None, + min_confidence: Any = None, router: Any = None, agent: Any = None, preset_builder: Callable[[str], dict] | None = None, @@ -700,8 +733,9 @@ def laya_preset( shape and is passed through untouched. The preset fixes the questions, not the route or the budget, so the per-call controls a - hand-written :func:`laya_predict` takes are available here too -- most usefully ``lang``, since - a preset's instructions are English text whatever state they read. + hand-written :func:`laya_predict` takes -- ``task``/``lang``/``max_len``/``head_max_len`` and + ``min_confidence`` -- are available here too; most usefully ``lang``, since a preset's + instructions are English text whatever state they read. """ preset_name = validate_preset(preset) state_d = validate_state(state) @@ -721,6 +755,7 @@ def laya_preset( lang=lang, max_len=max_len, head_max_len=head_max_len, + min_confidence=min_confidence, router=router, agent=agent, ) @@ -935,6 +970,7 @@ def laya_decide( schema: Any, model: Any = "auto", *, + min_confidence: Any = None, router: Any = None, agent: Any = None, ) -> dict: @@ -947,6 +983,11 @@ def laya_decide( answer is ``values`` -- enum members, integer levels, booleans -- beside the per-field ``confidence``/``probabilities`` and the usual routing/device/ latency metadata, so a client never parses an answer map by hand. + + ``min_confidence`` is ``laya.decide``'s own abstention control, not a generic + predict keyword: a field whose answer falls below it comes back as ``null`` in + ``values`` -- the raw answer stays in ``confidence``/``probabilities`` -- so a + client that trusts the values can tell "not confident" from a confident pick. """ # Lazy: keeps laya.structured (pure Python, but a module import is still a # module import) out of this module's import-time surface. @@ -954,6 +995,7 @@ def laya_decide( state_d = validate_state(state) model_name = validate_model(model) + min_conf = validate_min_confidence(min_confidence) try: # `decide` validates the schema itself (SchemaError names the offending # path) and projects the answers; calling it with return_details keeps @@ -962,14 +1004,16 @@ def laya_decide( if model_name == "auto": if router is None: raise ToolError("models_not_ready", "Router is not loaded (auto mode)") - details = decide(router, state_d, schema=schema, return_details=True) + details = decide(router, state_d, schema=schema, return_details=True, + min_confidence=min_conf) elif agent is not None: - details = decide(agent, state_d, schema=schema, return_details=True) + details = decide(agent, state_d, schema=schema, return_details=True, + min_confidence=min_conf) else: if router is None: raise ToolError("models_not_ready", "no agent/router loaded") details = decide(router, state_d, schema=schema, return_details=True, - model=model_name) + model=model_name, min_confidence=min_conf) except SchemaError as exc: raise ToolError("invalid_schema", str(exc)) from exc latency_ms = (time.perf_counter() - started) * 1000.0 diff --git a/tests/test_mcp.py b/tests/test_mcp.py index ce228995d..cb65b4839 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -44,6 +44,7 @@ validate_batch_requests, validate_budget, validate_lang, + validate_min_confidence, validate_model, validate_preset, validate_questions, @@ -1598,6 +1599,142 @@ def test_controls_signature_and_schema(): ok("schema/%s_required_unchanged" % name, required == (["preset", "state"] if name == "laya_preset" else ["state", "questions"]), repr(required)) + + +def test_min_confidence_control(): + """`min_confidence` -- core's per-call abstention gate (#361) -- is reachable over MCP. + + #567 forwarded every other per-call control (task/lang/max_len/head_max_len) to these tools, + but not this one: a client could pin a checkpoint and size its budget over the wire, yet could + not ask Laya to abstain on an unsure answer -- even though `min_confidence` is a validated + keyword of `Agent.predict`, `Router.predict`/`predict_batch` and `laya.decide` on `main`. + """ + import inspect + + # The range/bool/non-finite rule is core's; this layer only re-labels the refusal, so it can + # never be wider or narrower than `check_min_confidence`. + ok("minconf/none_is_unset", validate_min_confidence(None) is None) + for value in (0, 0.0, 0.5, 1, 1.0, 0.94): + ok("minconf/ok_%r" % (value,), validate_min_confidence(value) == float(value), + repr(validate_min_confidence(value))) + for bad in (-0.1, 1.1, 2, True, False, [], {}, "0.5", float("nan"), float("inf")): + expect_tool_error("minconf/rejected_%r" % (bad,), + lambda b=bad: validate_min_confidence(b), "invalid_min_confidence") + + # laya_predict forwards it to core's predict, and only when set: a no-abstention call reaches + # core exactly as it did before the keyword was exposed here. + router = ControlRouter() + laya_predict(STATE, QUESTIONS, router=router) + ok("predict/minconf_unset_absent", router.predict_calls == [{}], repr(router.predict_calls)) + router = ControlRouter() + laya_predict(STATE, QUESTIONS, min_confidence=0.85, router=router) + ok("predict/minconf_forwarded", router.predict_calls == [{"min_confidence": 0.85}], + repr(router.predict_calls)) + router = ControlRouter() + laya_predict(STATE, QUESTIONS, lang="de", max_len=1024, min_confidence=0.9, router=router) + ok("predict/minconf_alongside_controls", + router.predict_calls == [{"lang": "de", "max_len": 1024, "min_confidence": 0.9}], + repr(router.predict_calls)) + router = ControlRouter() + expect_tool_error("predict/bad_minconf", + lambda: laya_predict(STATE, QUESTIONS, min_confidence=2, router=router), + "invalid_min_confidence") + ok("predict/bad_minconf_no_call", router.predict_calls == [], repr(router.predict_calls)) + agent = ControlAgent() + laya_predict(STATE, QUESTIONS, model="english", min_confidence=0.5, agent=agent) + ok("predict/agent_minconf", agent.calls == [{"min_confidence": 0.5}], repr(agent.calls)) + + # laya_shortlist carries it to the answering pass beside the budget (route stays budget-free). + small = {"dept": {"type": "choice", "instructions": "pick", + "criteria": {"a": "A", "b": "B", "c": "C", "d": "D", "e": "E"}}} + router = ControlRouter(routed="multilingual") + laya_shortlist(STATE, small, k=2, head_max_len=384, min_confidence=0.8, + router=router, embed_fn=_tie_embed) + ok("shortlist/minconf_reaches_predict", + router.predict_calls == [{"model": "multilingual", "head_max_len": 384, + "min_confidence": 0.8}], repr(router.predict_calls)) + router = ControlRouter(routed="multilingual") + laya_shortlist(STATE, small, k=2, router=router, embed_fn=_tie_embed) + ok("shortlist/minconf_unset_absent", + "min_confidence" not in router.predict_calls[0], repr(router.predict_calls)) + router = ControlRouter() + expect_tool_error("shortlist/bad_minconf", + lambda: laya_shortlist(STATE, small, k=2, min_confidence=-1, router=router, + embed_fn=_tie_embed), + "invalid_min_confidence") + + # laya_preset inherits the control through laya_predict. + def builder(attr): + return {"probe": {"type": "noul", "instructions": "Does the `body` need a human?"}} + router = ControlRouter() + laya_preset("guard", STATE, min_confidence=0.7, router=router, preset_builder=builder) + ok("preset/minconf_forwarded", router.predict_calls == [{"min_confidence": 0.7}], + repr(router.predict_calls)) + + # The flagship: laya_decide uses core's abstention to null an unsure field while still + # reporting its confidence -- the one thing a values-consuming client needs and cannot get + # any other way over MCP. FakeRouter answers department at 0.94, urgency at 0.8, needs_human + # at 0.89, so a 0.9 gate keeps the first and nulls the other two. + router = TypeEchoRouter() + plain = laya_decide(STATE, DECIDE_SCHEMA, router=router) + ok("decide/minconf_unset_no_nul", plain["values"]["urgency"] == 2 + and plain["values"]["needs_human"] is True, repr(plain["values"])) + abstained = laya_decide(STATE, DECIDE_SCHEMA, min_confidence=0.9, router=router) + ok("decide/keeps_confident_field", abstained["values"]["department"] == "billing", + repr(abstained["values"])) + ok("decide/nulls_low_score", abstained["values"]["urgency"] is None, + repr(abstained["values"])) + ok("decide/nulls_low_noul", abstained["values"]["needs_human"] is None, + repr(abstained["values"])) + ok("decide/confidence_survives_null", + set(abstained["confidence"]) == {"department", "urgency", "needs_human"} + and abstained["confidence"]["urgency"] == 0.8, repr(abstained["confidence"])) + # A threshold of 0 abstains over nothing -- the raw answers are byte-identical to no call. + zero = laya_decide(STATE, DECIDE_SCHEMA, min_confidence=0.0, router=router) + ok("decide/zero_is_no_abstention", zero["values"] == plain["values"], repr(zero["values"])) + + # The tool is a thin surface over the documented core abstention, not a second projection. + import laya + + core = laya.decide(router, STATE, schema=DECIDE_SCHEMA, min_confidence=0.9) + ok("decide/abstention_parity_with_core", abstained["values"] == core, + "tool=%r core=%r" % (abstained["values"], core)) + + # Refused before core, so a bad threshold costs no forward pass. + router = TypeEchoRouter() + expect_tool_error("decide/bad_minconf", + lambda: laya_decide(STATE, DECIDE_SCHEMA, min_confidence=1.5, router=router), + "invalid_min_confidence") + + # Every tool that answers exposes the keyword, defaults it to None, and documents it. + tools = [laya_predict, laya_shortlist, laya_preset, laya_decide] + for fn in tools: + params = inspect.signature(fn).parameters + ok("sig/%s_has_min_confidence" % fn.__name__, "min_confidence" in params, + repr(sorted(params))) + ok("sig/%s_min_confidence_default_none" % fn.__name__, + params["min_confidence"].default is None, repr(params["min_confidence"].default)) + ok("sig/%s_min_confidence_keyword_only" % fn.__name__, + params["min_confidence"].kind is inspect.Parameter.KEYWORD_ONLY) + by_name = {t.name: t for t in asyncio.run(mcp_server.list_tools())} + for name in ("laya_predict", "laya_shortlist", "laya_preset", "laya_decide"): + tool = by_name[name] + schema = tool.input_schema if hasattr(tool, "input_schema") else tool.inputSchema + props = schema.get("properties", {}) + spec = props.get("min_confidence", {}) + ok("schema/%s_exposes_min_confidence" % name, "min_confidence" in props, + repr(sorted(props))) + ok("schema/%s_min_confidence_nullable" % name, + {"type": "null"} in (spec.get("anyOf") or []), repr(spec)) + ok("schema/%s_min_confidence_typed" % name, + any(opt.get("type") in ("number", "integer") for opt in (spec.get("anyOf") or [spec])), + repr(spec)) + ok("schema/%s_min_confidence_not_required" % name, + "min_confidence" not in schema.get("required", []), repr(schema.get("required"))) + ok("schema/%s_desc_documents_min_confidence" % name, + "min_confidence" in tool.description.lower(), repr(tool.description[-80:])) + + def test_question_validation_matches_the_agent(): """MCP must reject a bad question the same way the agent does, and say so. @@ -1880,6 +2017,7 @@ def test_server_registration(): test_controls_shortlist() test_controls_preset() test_controls_signature_and_schema() +test_min_confidence_control() test_question_validation_matches_the_agent() test_a_bad_question_is_a_caller_error_not_a_server_fault() test_timeout_removed()