From 70367cd96bf825e4d8c1f15b87e694cff10d39ec Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:58:46 +0545 Subject: [PATCH 1/7] feat(agent): per-call hooks on predict_long, and no window for a skipped answer predict_long was the only inference entry point that accepted none of the per-call hook arguments, so a hook installed per call had to be promoted to agent state just to instrument a long-document scan. The hooks wrap the inference that answers the state: one call on the fits-in-one-window path, one shared predict_batch over the windows otherwise. A start hook that calls ctx.skip(...) is answering the document, not a window, so its payload is returned unattributed with usage["windows"] = 0 instead of being aggregated into a deciding-window entry for windows that never scored it. --- README.md | 6 +++ docs/hooks/api.md | 11 ++++++ laya/agent.py | 44 ++++++++++++++++++++-- tests/test_hooks_api.py | 8 ++++ tests/test_predict_long.py | 75 +++++++++++++++++++++++++++++++++++--- 5 files changed, 136 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 54c20cf9a..c339df947 100644 --- a/README.md +++ b/README.md @@ -570,12 +570,18 @@ per question: ```python result = agent.predict_long(state, questions) # windows the state, one result back result = agent.predict_long(state, questions, window=256) # smaller window isolates a localized span +result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan, instrumented ``` - `noul` takes the strongest window (the statement holds if any window supports it). - `choice` / `score` take the most-confident window — averaging over a long, mostly-neutral document lets the neutral majority out-vote the one window that saw the deciding span. - A state that already fits one window is passed straight to `system_one` (identical output). +- Hooks wrap the inference that answers the state, so on a scanned document `on_predict_start` + fires once with `ctx.states` holding the decoded windows, not the state you passed in (it was + tokenized to produce them). A start hook that answers with `ctx.skip(...)` is answering the + whole document: its result comes back with no `answer["window"]` and `usage["windows"]` at 0, + because no window scored it. A smaller `window` isolates a short deciding span better (it becomes a larger fraction of its window); the default (`max_len - head_max_len`) favors context and throughput. Output shape matches diff --git a/docs/hooks/api.md b/docs/hooks/api.md index 8817e6663..fc14b7337 100644 --- a/docs/hooks/api.md +++ b/docs/hooks/api.md @@ -229,11 +229,22 @@ agent.system_one(state, questions, hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None, hooks_timeout=None, max_len=None, head_max_len=None) +agent.predict_long(state, questions, window=None, stride=None, aggregate="auto", + batch_size=None, lang=None, + hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None, + hooks_timeout=None) + agent.predict(...) # alias of system_one ``` - `hooks_raise` and `hooks_timeout` on a per-call method default to `None`, meaning "use the instance value". - `hooks_concurrent` is instance-level only. +- On `predict_long` the hooks wrap the inference that answers the state, which for a document + needing several windows is the single shared `predict_batch` over them: `on_predict_start` fires + once, and `ctx.states` holds the decoded window texts in scan order rather than the caller's + state, which was tokenized to produce them. A start hook that answers with `ctx.skip(...)` is + answering the document, so its one result is returned as-is with no `window` key and + `usage["windows"]` at `0` -- no window scored it. ### Router diff --git a/laya/agent.py b/laya/agent.py index 1b58de6f3..5cc8d044b 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -956,7 +956,10 @@ def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[st def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]], window: Optional[int] = None, stride: Optional[int] = None, aggregate: str = "auto", batch_size: Optional[int] = None, - lang: Optional[str] = None) -> Dict[str, Any]: + lang: Optional[str] = None, + hooks=None, on_predict_start=None, on_predict_end=None, + hooks_raise: Optional[bool] = None, + hooks_timeout: Optional[float] = None) -> Dict[str, Any]: """Evaluate questions over a state longer than the context window, scanning it in overlapping windows and aggregating per question. @@ -980,6 +983,13 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ A state that already fits one window is passed straight to `system_one` (identical output). + The hooks wrap the inference that answers the state, which for a document needing several + windows is the one shared `predict_batch` over them: `on_predict_start` fires once, and + `ctx.states` holds the decoded window texts in scan order -- not the caller's `state`, which + was tokenized to produce them. A start hook that rewrites `ctx.states` therefore rewrites + windows, and one that calls `ctx.skip(...)` is answering the document rather than a window, + so its payload is returned without window attribution. + Args: window: state tokens per window. Defaults to the per-question state budget (`max_len - head_max_len - 8`) -- the most a window can hold for every question. @@ -993,11 +1003,20 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ aggregate: "auto" (the per-type rules above) is the only mode for now. batch_size: cap on windows per forward pass, to bound memory on very long states. lang: per-language temperature selection, as in `system_one`. + hooks (HookArg): Per-call hooks, appended after any installed on the Agent. + See `laya.hooks`. + on_predict_start (PredictHookArg): A per-call start hook, as in `system_one`. + on_predict_end (PredictHookArg): A per-call end hook, as in `system_one`. + hooks_raise: Override the Agent's `hooks_raise` for this call. + hooks_timeout: Override the Agent's `hooks_timeout` for this call. Returns a single result dict, the same shape as `system_one`, with `usage["windows"]` added. """ if aggregate != "auto": raise ValueError("predict_long: only aggregate='auto' is supported") + hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start, + "on_predict_end": on_predict_end, "hooks_raise": hooks_raise, + "hooks_timeout": hooks_timeout} max_len = self.cfg.get("max_len", 512) head_max_len = self.cfg.get("head_max_len", 192) budget = window if (window and window > 0) else max(64, max_len - head_max_len - 8) @@ -1006,7 +1025,7 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ add_special_tokens=False)["input_ids"] # Fits in one window: identical to a plain call, no windowing overhead. if len(state_ids) <= budget: - return self.system_one(state, questions, lang=lang) + return self.system_one(state, questions, lang=lang, **hook_kwargs) step = stride if (stride and stride > 0) else max(1, budget // 2) windows, starts = [], [] @@ -1021,7 +1040,26 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ break i += step - results = self.predict_batch(windows, questions, batch_size=batch_size, lang=lang) + results = self.predict_batch(windows, questions, batch_size=batch_size, lang=lang, + **hook_kwargs) + + if len(results) != len(windows): + # Only a hook can change the count: `ctx.skip(...)` runs before the split, so a hook + # answering the whole document has no way to know how many windows it will produce and + # returns one result. Aggregating over that payload would pick between answers that were + # never scored and name a deciding window that decided nothing, so pass the document + # answer through unattributed and say which windows were scored: none of them. + if len(results) != 1: + raise ValueError( + "predict_long: %d windows scored, %d results returned; a start hook that answers" + " a long state returns one result for the document" % (len(windows), len(results))) + warnings.warn("laya: predict_long: a hook answered the state before it was scanned, so " + "no window decided the result and none is reported", RuntimeWarning, + stacklevel=2) + document = dict(results[0]) + document["usage"] = dict(document.get("usage") or {}) + document["usage"]["windows"] = 0 + return document ids = list(questions.keys()) internal = {qid: self._to_internal(questions[qid]) for qid in ids} diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py index fa750e30b..7dc6422eb 100644 --- a/tests/test_hooks_api.py +++ b/tests/test_hooks_api.py @@ -72,6 +72,7 @@ def check_param(name, fn, param, default, kind=None): # --------------------------------------------------------------- predict surfaces for label, fn in (("Agent.predict_batch", Agent.predict_batch), ("Agent.system_one", Agent.system_one), + ("Agent.predict_long", Agent.predict_long), ("Router.predict", Router.predict), ("ONNXAgent.system_one", ONNXAgent.system_one)): check_param(label, fn, "hooks", None) @@ -80,6 +81,13 @@ def check_param(name, fn, param, default, kind=None): check_param(label, fn, "hooks_raise", None) check_param(label, fn, "hooks_timeout", None) +# The scan sizes its windows from the checkpoint budget, so it takes `window`/`stride` instead +# of the per-call token overrides the single-window entries take. +check_param("Agent.predict_long", Agent.predict_long, "window", None) +check_param("Agent.predict_long", Agent.predict_long, "stride", None) +for param in ("max_len", "head_max_len"): + check("Agent.predict_long/%s not accepted" % param, param in sig(Agent.predict_long), False) + check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None) # per-call token-budget overrides diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index e8d70688f..d02eb3bc0 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -2,11 +2,13 @@ Weight-free. The real forward path is stubbed (predict_batch returns canned per-window answers), so this checks only predict_long's own logic: the fits-in-one-window short-circuit, the overlapping -window split, and the per-type aggregation (noul = strongest window, choice/score = most-confident -window). Numerical behaviour on real weights is exercised in tests/test_local_e2e.py. +window split, the per-type aggregation (noul = strongest window, choice/score = most-confident +window), and the per-call hook controls it forwards to whichever of those two calls runs. Numerical +behaviour on real weights is exercised in tests/test_local_e2e.py. """ import os import sys +import warnings sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -46,14 +48,17 @@ def make_agent(batch_result_fn): a.cfg = {"max_len": 100, "head_max_len": 20} # budget = max(64, 100-20-8) = 72 a.tok = _Tok() a._to_internal = staticmethod(Agent._to_internal).__func__ - a._calls = {"system_one": 0, "batch_states": None} + a._calls = {"system_one": 0, "batch_states": None, "system_one_kwargs": None, + "batch_kwargs": None} - def _system_one(state, questions, lang=None): + def _system_one(state, questions, lang=None, **controls): a._calls["system_one"] += 1 + a._calls["system_one_kwargs"] = controls return {"model": "laya-rl-agent", "answers": {"_via": "system_one"}, "usage": {"input_tokens": 1}} - def _predict_batch(states, questions, batch_size=None, lang=None): + def _predict_batch(states, questions, batch_size=None, lang=None, **controls): a._calls["batch_states"] = list(states) + a._calls["batch_kwargs"] = controls return batch_result_fn(list(states), questions) a.system_one = _system_one @@ -108,6 +113,66 @@ def canned(states, q): check_raises("aggregate/rejects unknown mode", ValueError, lambda: a.predict_long({"body": "y" * 300}, Q, aggregate="mean")) +# 4. the per-call hook controls `predict`/`system_one`/`predict_batch` take reach the scan +HOOK_KEYS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise", "hooks_timeout") +LONG = {"body": "y" * 300} +sentinel = object() +a = make_agent(canned) +res = a.predict_long(LONG, Q, hooks=[sentinel], hooks_raise=False, hooks_timeout=2.5) +got = a._calls["batch_kwargs"] +check("hooks/forwarded to the scan", sorted(got), sorted(HOOK_KEYS)) +check("hooks/the hook list reaches it", got.get("hooks"), [sentinel]) +check("hooks/hooks_raise reaches it", got.get("hooks_raise"), False) +check("hooks/hooks_timeout reaches it", got.get("hooks_timeout"), 2.5) +check("hooks/start/end default to None", + [got.get("on_predict_start"), got.get("on_predict_end")], [None, None]) + +# 5. the same controls reach the fits-in-one-window path +a = make_agent(lambda s, q: []) +a.predict_long({"body": "x" * 50}, Q, hooks=[sentinel], hooks_timeout=1.25) +got = a._calls["system_one_kwargs"] +check("hooks/forwarded to system_one", sorted(got), sorted(HOOK_KEYS)) +check("hooks/the hook list reaches it there", got.get("hooks"), [sentinel]) +check("hooks/hooks_timeout reaches it there", got.get("hooks_timeout"), 1.25) + +# 6. what the scan forwards is the window texts, in scan order, not the caller's state object +a = make_agent(canned) +a.predict_long(LONG, Q) +states = a._calls["batch_states"] +check("states/several windows", len(states) > 1, True) +check("states/decoded text, not the caller's dict", + all(isinstance(s, str) for s in states) and states[0] == "w0_71", True) +check("states/they overlap", a._calls["batch_states"][1], "w36_107") + +# 7. forwarding the controls must not move a decision +plain = make_agent(canned).predict_long(LONG, Q) +with_hooks = make_agent(canned).predict_long(LONG, Q, hooks=[sentinel], hooks_timeout=9) +check("hooks/forwarding is decision-neutral", with_hooks, plain) + +# 8. a start hook that answers the document is not attributed to a window that was never scored +def one_document_answer(states, q): + return [{"answers": {"dept": {"type": "choice", "choice": "a", "answer_confidence": 0.7}, + "flag": {"type": "noul", "noul": 0.2, "answer_confidence": 0.8}}, + "usage": {"input_tokens": 3, "output_tokens": 0}}] + + +a = make_agent(one_document_answer) +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + res = a.predict_long(LONG, Q) +nwin = len(a._calls["batch_states"]) +check("skip/the state was split into several windows", nwin > 1, True) +check("skip/no window was scored", res["usage"]["windows"], 0) +check("skip/the payload tokens are kept", res["usage"]["input_tokens"], 3) +check("skip/no deciding window is claimed", + sorted(k for v in res["answers"].values() for k in v if k == "window"), []) +check("skip/the hook's answer is returned", res["answers"]["dept"]["choice"], "a") +check("skip/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"]) + +# 9. a payload that matches neither the document nor the windows is an error, not a guess +a = make_agent(lambda s, q: [{"answers": {}, "usage": {}}] * 2) +check_raises("skip/rejects a count matching nothing", ValueError, lambda: a.predict_long(LONG, Q)) + print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) for f in FAIL: print(" FAIL " + f) From ea222a6113a408d2a8a194a020b42f431d0a0874 Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:01:24 +0545 Subject: [PATCH 2/7] fix(agent): make usage["windows"] total on every predict_long path A state that fits one window returned no windows key at all, so the natural gate (usage["windows"] == 0) raised KeyError on the shortest, most common inputs and could not tell "a hook answered it" from "it was scanned". predict_long now reports 1 there, keeping 0 for a hook that answered a multi-window document and N for a scan, and the copy-then-write means a start hook's own payload dict is never mutated. --- README.md | 4 +++- docs/hooks/api.md | 5 ++++- laya/agent.py | 13 +++++++++++-- tests/test_predict_long.py | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c339df947..e1d8a8b79 100644 --- a/README.md +++ b/README.md @@ -576,7 +576,9 @@ result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan, - `noul` takes the strongest window (the statement holds if any window supports it). - `choice` / `score` take the most-confident window — averaging over a long, mostly-neutral document lets the neutral majority out-vote the one window that saw the deciding span. -- A state that already fits one window is passed straight to `system_one` (identical output). +- A state that already fits one window is passed straight to `system_one` (identical output, plus + `usage["windows"] = 1`). The key is total: `1` single window, `N` scanned windows, `0` a hook + answered a multi-window document before the model read any of it. - Hooks wrap the inference that answers the state, so on a scanned document `on_predict_start` fires once with `ctx.states` holding the decoded windows, not the state you passed in (it was tokenized to produce them). A start hook that answers with `ctx.skip(...)` is answering the diff --git a/docs/hooks/api.md b/docs/hooks/api.md index fc14b7337..e6d904d20 100644 --- a/docs/hooks/api.md +++ b/docs/hooks/api.md @@ -244,7 +244,10 @@ agent.predict(...) # alias of system_one once, and `ctx.states` holds the decoded window texts in scan order rather than the caller's state, which was tokenized to produce them. A start hook that answers with `ctx.skip(...)` is answering the document, so its one result is returned as-is with no `window` key and - `usage["windows"]` at `0` -- no window scored it. + `usage["windows"]` at `0` -- no window scored it. `usage["windows"]` is total over the three + paths: `0` here, `1` for a state that fit a single window (a hook that answered it is answering + that one window, so the count is the same as the model's), and `N` for a document scanned in `N` + overlapping windows. ### Router diff --git a/laya/agent.py b/laya/agent.py index 5cc8d044b..c7407a312 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -1011,6 +1011,9 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ hooks_timeout: Override the Agent's `hooks_timeout` for this call. Returns a single result dict, the same shape as `system_one`, with `usage["windows"]` added. + The key is always present and counts the windows the answer was read from: `1` for a state + that fit one window, `N` for a document scanned in `N` overlapping windows, and `0` when a + start hook answered a multi-window document before any window was scored. """ if aggregate != "auto": raise ValueError("predict_long: only aggregate='auto' is supported") @@ -1023,9 +1026,15 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ state_ids = self.tok(serialize_state(state).replace(self.tok.mask_token, " "), add_special_tokens=False)["input_ids"] - # Fits in one window: identical to a plain call, no windowing overhead. + # Fits in one window: identical to a plain call, no windowing overhead. `windows` is still + # written, so the key is total over the three paths this method can take and a caller can + # ask "how much of the document did the model read?" without handling a KeyError on the + # shortest, most common inputs. The result is copied first: a start hook that answers with + # `ctx.skip(...)` hands back its own payload dict, and it may be a cached object. if len(state_ids) <= budget: - return self.system_one(state, questions, lang=lang, **hook_kwargs) + single = dict(self.system_one(state, questions, lang=lang, **hook_kwargs)) + single["usage"] = {**(single.get("usage") or {}), "windows": 1} + return single step = stride if (stride and stride > 0) else max(1, budget // 2) windows, starts = [], [] diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index d02eb3bc0..a174b6898 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -7,6 +7,7 @@ behaviour on real weights is exercised in tests/test_local_e2e.py. """ import os +import re import sys import warnings @@ -74,6 +75,17 @@ def _predict_batch(states, questions, batch_size=None, lang=None, **controls): short = a.predict_long({"body": "x" * 50}, Q) # 50 tokens <= budget 72 check("short/delegates to system_one", short["answers"], {"_via": "system_one"}) check("short/no predict_batch call", a._calls["batch_states"], None) +check("short/one window is reported", short["usage"]["windows"], 1) +check("short/system_one's own usage is kept", short["usage"]["input_tokens"], 1) + +# 1b. the count is added to a copy, because a start hook that skips hands back the caller's own +# payload dict and that object may be cached and reused +payload = {"model": "laya-rl-agent", "answers": {"_via": "system_one"}, "usage": {"input_tokens": 1}} +a = make_agent(lambda s, q: []) +a.system_one = lambda state, questions, lang=None, **controls: payload +via_hook = a.predict_long({"body": "x" * 50}, Q) +check("short/a new result dict comes back", via_hook is payload, False) +check("short/the caller's payload is not written to", payload["usage"], {"input_tokens": 1}) # 2. long state -> overlapping windows, aggregated per question def canned(states, q): @@ -173,6 +185,31 @@ def one_document_answer(states, q): a = make_agent(lambda s, q: [{"answers": {}, "usage": {}}] * 2) check_raises("skip/rejects a count matching nothing", ValueError, lambda: a.predict_long(LONG, Q)) +# 10. the count is total, so one question works on every result predict_long can return +short_all = make_agent(lambda s, q: []).predict_long({"body": "x" * 50}, Q) +scan_agent = make_agent(canned) +scanned_all = scan_agent.predict_long(LONG, Q) +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + hook_all = make_agent(one_document_answer).predict_long(LONG, Q) +check("total/no path leaves the key out", + [r["usage"].get("windows", "") for r in (short_all, scanned_all, hook_all)], + [1, len(scan_agent._calls["batch_states"]), 0]) +check("total/0 separates a hook answer from a single-window one", + (hook_all["usage"]["windows"], short_all["usage"]["windows"]), (0, 1)) + + +# 11. the page that teaches this key teaches the value the code actually writes +README = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "README.md"), encoding="utf-8").read() +bullet = README[README.index("A state that already fits one window"):][:600] +check("docs/README pins the single-window value", 'usage["windows"] = 1' in bullet, True) +check("docs/README states the key is total", "The key is total" in bullet, True) +check("docs/README documents all three counts", sorted(set(re.findall(r"`([0-9N])`", bullet))), + ["0", "1", "N"]) +check("docs/predict_long's docstring says the key is total", + "always present" in (Agent.predict_long.__doc__ or ""), True) + print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) for f in FAIL: print(" FAIL " + f) From c78bac4cabe470708acbe38463c63e2af25e62a8 Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:02:39 +0545 Subject: [PATCH 3/7] test(agent): name the check that fails when the count is missing --- tests/test_predict_long.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index a174b6898..1ba54c84a 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -75,8 +75,8 @@ def _predict_batch(states, questions, batch_size=None, lang=None, **controls): short = a.predict_long({"body": "x" * 50}, Q) # 50 tokens <= budget 72 check("short/delegates to system_one", short["answers"], {"_via": "system_one"}) check("short/no predict_batch call", a._calls["batch_states"], None) -check("short/one window is reported", short["usage"]["windows"], 1) -check("short/system_one's own usage is kept", short["usage"]["input_tokens"], 1) +check("short/one window is reported", short["usage"].get("windows", ""), 1) +check("short/system_one's own usage is kept", short["usage"].get("input_tokens", ""), 1) # 1b. the count is added to a copy, because a start hook that skips hands back the caller's own # payload dict and that object may be cached and reused From 9a52a2241a65af68e5f1daecb7b83ce27ce2a113 Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:03:11 +0545 Subject: [PATCH 4/7] test(agent): keep the totality check's failure message readable --- tests/test_predict_long.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index 1ba54c84a..59d280da8 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -196,7 +196,8 @@ def one_document_answer(states, q): [r["usage"].get("windows", "") for r in (short_all, scanned_all, hook_all)], [1, len(scan_agent._calls["batch_states"]), 0]) check("total/0 separates a hook answer from a single-window one", - (hook_all["usage"]["windows"], short_all["usage"]["windows"]), (0, 1)) + (hook_all["usage"].get("windows", ""), short_all["usage"].get("windows", "")), + (0, 1)) # 11. the page that teaches this key teaches the value the code actually writes From bd18c4714f605014237187022f6fe20adc7ea855 Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:47:07 +0545 Subject: [PATCH 5/7] fix(agent): take a hook's answer from ctx.results, not from the result count A start hook that answered a long state and a start hook that replaced the window list both return fewer results than there were windows, and the count could not tell them apart: the rewrite was reported to the caller as "a hook answered before the state was scanned", and the tokens it did book were dropped. predict_long now appends its own recorder after the caller's start hooks and reads ctx.results from the engine's own context, so only a real answer takes the unattributed path and a rewritten list raises with the split it could not attribute. The same rule covers the fits-in-one-window path, where a hook answer now reports 0 windows like every other hook answer, and the window list is copied before the hooks see it, so an in-place append can no longer move the split the answers are attributed to. --- README.md | 10 +- docs/hooks/api.md | 13 ++- laya/agent.py | 94 ++++++++++++---- tests/test_predict_long.py | 212 +++++++++++++++++++++++++++++++++---- 4 files changed, 278 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index e1d8a8b79..c87cb1f87 100644 --- a/README.md +++ b/README.md @@ -578,12 +578,14 @@ result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan, document lets the neutral majority out-vote the one window that saw the deciding span. - A state that already fits one window is passed straight to `system_one` (identical output, plus `usage["windows"] = 1`). The key is total: `1` single window, `N` scanned windows, `0` a hook - answered a multi-window document before the model read any of it. + answered the document before the model read any of it. - Hooks wrap the inference that answers the state, so on a scanned document `on_predict_start` fires once with `ctx.states` holding the decoded windows, not the state you passed in (it was - tokenized to produce them). A start hook that answers with `ctx.skip(...)` is answering the - whole document: its result comes back with no `answer["window"]` and `usage["windows"]` at 0, - because no window scored it. + tokenized to produce them). A start hook may rewrite those window texts, and what is scored is + what it leaves behind, but it may not add, drop or reorder windows: the answer attributes itself + to a span of your document by index, so a call that runs inference over any other number of + states raises. A hook that means to answer the document calls `ctx.skip(...)` instead: its result + comes back with no `answer["window"]` and `usage["windows"]` at 0, because no window scored it. A smaller `window` isolates a short deciding span better (it becomes a larger fraction of its window); the default (`max_len - head_max_len`) favors context and throughput. Output shape matches diff --git a/docs/hooks/api.md b/docs/hooks/api.md index e6d904d20..0f5d83d96 100644 --- a/docs/hooks/api.md +++ b/docs/hooks/api.md @@ -242,12 +242,15 @@ agent.predict(...) # alias of system_one - On `predict_long` the hooks wrap the inference that answers the state, which for a document needing several windows is the single shared `predict_batch` over them: `on_predict_start` fires once, and `ctx.states` holds the decoded window texts in scan order rather than the caller's - state, which was tokenized to produce them. A start hook that answers with `ctx.skip(...)` is - answering the document, so its one result is returned as-is with no `window` key and + state, which was tokenized to produce them. A start hook may rewrite the text of those windows, + and what is scored is what it leaves behind, but it may not change how many there are or their + order -- the answer attributes itself to a span of the caller's document by index, so a call that + runs inference over any other number of states raises. A hook that means to answer the document + calls `ctx.skip(...)` instead: its one result is returned as-is with no `window` key and `usage["windows"]` at `0` -- no window scored it. `usage["windows"]` is total over the three - paths: `0` here, `1` for a state that fit a single window (a hook that answered it is answering - that one window, so the count is the same as the model's), and `N` for a document scanned in `N` - overlapping windows. + paths: `0` for a hook answer on either path, so a cached answer never reads as a window the model + read, `1` for a state that fit a single window and was scored, and `N` for a document scanned in + `N` overlapping windows. ### Router diff --git a/laya/agent.py b/laya/agent.py index c7407a312..c026f589d 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -29,8 +29,8 @@ temp_bucket, ) from .hooks import ( - HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks, - validate_timeout, + HookRegistry, PredictContext, _as_sequence, aggregate_usage, compose_hooks, dispatch, + normalise_hooks, validate_timeout, ) from .revisions import resolve_revision, snapshot_revision, verify_digests @@ -216,6 +216,34 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype: return amp_dtype(checkpoint_default) +def _start_evidence(): + """A recorder for `predict_long`: what the start-hook chain did, rather than what it cost. + + Why a recorder at all: the result count cannot tell a hook that answered the document from a + hook that replaced the window list. Both return fewer results than there were windows, and the + two need opposite readings -- the first scored no window, the second scored the ones the hook + left behind, and booked tokens for them. `ctx.results` already distinguishes them, and only the + engine's own context sees it, so `predict_long` appends this probe to the call's start hooks. + + Returns the probe and the dict it fills: `answered` is whether a hook replaced the call before + inference, `states` is the number of states that reached it. + """ + evidence = {"answered": False, "states": None} + + def probe(ctx): + evidence["answered"] = ctx.results is not None + evidence["states"] = len(ctx.states) + + return probe, evidence + + +def _with_start_probe(hook_kwargs, probe): + """`hook_kwargs` with `probe` appended after the caller's own start hooks.""" + kwargs = dict(hook_kwargs) + kwargs["on_predict_start"] = list(_as_sequence(hook_kwargs.get("on_predict_start"))) + [probe] + return kwargs + + class Agent(HookRegistry): """System 1 decision model runtime: fast, non-autoregressive, calibrated decisions.""" @@ -986,9 +1014,11 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ The hooks wrap the inference that answers the state, which for a document needing several windows is the one shared `predict_batch` over them: `on_predict_start` fires once, and `ctx.states` holds the decoded window texts in scan order -- not the caller's `state`, which - was tokenized to produce them. A start hook that rewrites `ctx.states` therefore rewrites - windows, and one that calls `ctx.skip(...)` is answering the document rather than a window, - so its payload is returned without window attribution. + was tokenized to produce them. A start hook may rewrite the text of those windows, and what + is scored is what it leaves behind. It may not change how many there are, or their order: + the per-window attribution names spans of the caller's document by index, so a different + count raises. A hook that means to answer the document itself calls `ctx.skip(...)`, whose + single payload is returned without window attribution. Args: window: state tokens per window. Defaults to the per-question state budget @@ -1011,9 +1041,10 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ hooks_timeout: Override the Agent's `hooks_timeout` for this call. Returns a single result dict, the same shape as `system_one`, with `usage["windows"]` added. - The key is always present and counts the windows the answer was read from: `1` for a state - that fit one window, `N` for a document scanned in `N` overlapping windows, and `0` when a - start hook answered a multi-window document before any window was scored. + The key is always present and counts the windows the model scored to produce the answer: `1` + for a state that fit one window, `N` for a document scanned in `N` overlapping windows, and + `0` when a start hook answered before any window was scored -- on either path, so a cached + answer never reads as a window the model read. """ if aggregate != "auto": raise ValueError("predict_long: only aggregate='auto' is supported") @@ -1032,8 +1063,13 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ # shortest, most common inputs. The result is copied first: a start hook that answers with # `ctx.skip(...)` hands back its own payload dict, and it may be a cached object. if len(state_ids) <= budget: - single = dict(self.system_one(state, questions, lang=lang, **hook_kwargs)) - single["usage"] = {**(single.get("usage") or {}), "windows": 1} + probe, evidence = _start_evidence() + single = dict(self.system_one(state, questions, lang=lang, + **_with_start_probe(hook_kwargs, probe))) + # Why 0 for a hook answer here: the state did fit one window, but no window was scored, + # which is the same fact the multi-window path reports as 0. Reading 1 would make a + # cached answer and a served answer agree on how much of the input the model saw. + single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1} return single step = stride if (stride and stride > 0) else max(1, budget // 2) @@ -1049,19 +1085,22 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ break i += step - results = self.predict_batch(windows, questions, batch_size=batch_size, lang=lang, - **hook_kwargs) - - if len(results) != len(windows): - # Only a hook can change the count: `ctx.skip(...)` runs before the split, so a hook - # answering the whole document has no way to know how many windows it will produce and - # returns one result. Aggregating over that payload would pick between answers that were - # never scored and name a deciding window that decided nothing, so pass the document - # answer through unattributed and say which windows were scored: none of them. + probe, evidence = _start_evidence() + # `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that + # mutates it in place (`append`, `sort`) would otherwise also grow the split this call + # attributes answers to, and the counts would agree while `starts` no longer lined up. + results = self.predict_batch(list(windows), questions, batch_size=batch_size, lang=lang, + **_with_start_probe(hook_kwargs, probe)) + + if evidence["answered"]: + # The hook replaced the call before any window was scored. Aggregating over its payload + # would pick between answers that were never scored and name a deciding window that + # decided nothing, so pass the document answer through unattributed and report the + # count as what it is: none of them. if len(results) != 1: raise ValueError( - "predict_long: %d windows scored, %d results returned; a start hook that answers" - " a long state returns one result for the document" % (len(windows), len(results))) + "predict_long: a start hook answered this state with %d results; ctx.skip()" + " takes one result for the document, not one per window" % len(results)) warnings.warn("laya: predict_long: a hook answered the state before it was scanned, so " "no window decided the result and none is reported", RuntimeWarning, stacklevel=2) @@ -1070,6 +1109,19 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ document["usage"]["windows"] = 0 return document + if len(results) != len(windows): + # A hook can also rewrite `ctx.states` and let inference run on what it left behind, + # which moves the count without answering anything. Those results cannot be attributed + # to windows of the caller's document -- the states that produced them are the hook's + # -- so refuse rather than name a span that was never read. + survived = ("" if evidence["states"] is None else + " (%d states reached inference)" % evidence["states"]) + raise ValueError( + "predict_long: the state was split into %d windows and the call returned %d" + " results%s; a start hook may rewrite the text of a window but not their number" + " or their order -- call ctx.skip([result]) to answer the document without scoring" + " it" % (len(windows), len(results), survived)) + ids = list(questions.keys()) internal = {qid: self._to_internal(questions[qid]) for qid in ids} answers = {} diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index 59d280da8..ab4471ddc 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -9,10 +9,13 @@ import os import re import sys +import threading import warnings sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import numpy as np # noqa: E402 + from laya.agent import Agent # noqa: E402 PASS, FAIL = [], [] @@ -33,6 +36,10 @@ def check_raises(name, exc, fn): FAIL.append("%s: did not raise %s" % (name, exc.__name__)) +def check_true(name, cond, detail=""): + (PASS if cond else FAIL).append(name if cond else "%s: %s" % (name, detail)) + + class _Tok: mask_token = "[M]" @@ -67,6 +74,49 @@ def _predict_batch(states, questions, batch_size=None, lang=None, **controls): return a +def make_real_agent(): + """`predict_long` on the real `predict_batch`/`system_one`, with only the three composed + helpers stubbed -- the harness `tests/test_hooks.py` uses, so no weights are involved. + + Why both agents: the stubbed `predict_batch` above is the right instrument for what + `predict_long` *forwards*, but it never runs a hook, so anything it says about a hook answering + rests on a result count alone. Here the hook chain really runs and every forward pass is + counted, which is the difference between "a hook answered the document" and "a hook replaced + the window list" -- both change the count. + """ + a = Agent.__new__(Agent) + a.cfg = {"max_len": 100, "head_max_len": 20} # budget = max(64, 100-20-8) = 72 + a.tok = type("Tok", (_Tok,), {"pad_token_id": 0})() + a._to_internal = staticmethod(Agent._to_internal).__func__ + a.model_id = "convaiinnovations/laya" + a.hooks = [] + a.hooks_raise = True + a.hooks_timeout = None + a._hooks_lock = threading.Lock() + a._forward_calls, a._encoded = [], [] + + def _encode_state(state, ids, internal, **overrides): + a._encoded.append(state) + return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids] + + def _forward(b): + n = b["input_ids"].shape[0] + a._forward_calls.append(n) + return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32) + + def _decode_answers(logits, act, items, ids, internal, row, **kw): + conf = 0.4 + 0.05 * row # so the last window decides + return {"dept": {"type": "choice", "choice": "a", "confidence": conf, + "answer_confidence": conf}, + "flag": {"type": "noul", "noul": conf, "confidence": conf, + "answer_confidence": conf}} + + a._encode_state = _encode_state + a._forward = _forward + a._decode_answers = _decode_answers + return a + + Q = {"dept": {"type": "choice", "instructions": "?", "criteria": {"a": "x", "b": "y"}}, "flag": {"type": "noul", "instructions": "?"}} @@ -129,15 +179,25 @@ def canned(states, q): HOOK_KEYS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise", "hooks_timeout") LONG = {"body": "y" * 300} sentinel = object() + + +def sentinel_start(ctx): + pass + + a = make_agent(canned) -res = a.predict_long(LONG, Q, hooks=[sentinel], hooks_raise=False, hooks_timeout=2.5) +res = a.predict_long(LONG, Q, hooks=[sentinel], on_predict_start=sentinel_start, + hooks_raise=False, hooks_timeout=2.5) got = a._calls["batch_kwargs"] check("hooks/forwarded to the scan", sorted(got), sorted(HOOK_KEYS)) check("hooks/the hook list reaches it", got.get("hooks"), [sentinel]) check("hooks/hooks_raise reaches it", got.get("hooks_raise"), False) check("hooks/hooks_timeout reaches it", got.get("hooks_timeout"), 2.5) -check("hooks/start/end default to None", - [got.get("on_predict_start"), got.get("on_predict_end")], [None, None]) +started = got.get("on_predict_start") +started = started if isinstance(started, list) else ([] if started is None else [started]) +check("hooks/the caller's start hook still runs first", + bool(started) and started[0] is sentinel_start, True) +check("hooks/end stays absent when the caller sets none", got.get("on_predict_end"), None) # 5. the same controls reach the fits-in-one-window path a = make_agent(lambda s, q: []) @@ -161,29 +221,123 @@ def canned(states, q): with_hooks = make_agent(canned).predict_long(LONG, Q, hooks=[sentinel], hooks_timeout=9) check("hooks/forwarding is decision-neutral", with_hooks, plain) -# 8. a start hook that answers the document is not attributed to a window that was never scored -def one_document_answer(states, q): - return [{"answers": {"dept": {"type": "choice", "choice": "a", "answer_confidence": 0.7}, - "flag": {"type": "noul", "noul": 0.2, "answer_confidence": 0.8}}, - "usage": {"input_tokens": 3, "output_tokens": 0}}] +# 8. a start hook that answers the document, on the engine's own evidence +DOC = {"model": "laya-rl-agent", + "answers": {"dept": {"type": "choice", "choice": "b", "answer_confidence": 0.9}, + "flag": {"type": "noul", "noul": 0.2, "answer_confidence": 0.8}}, + "usage": {"input_tokens": 3, "output_tokens": 0}} + +# The control first: how many windows this state really splits into, measured from the forward +# passes it costs rather than from the key under test. Two questions, one row each per window. +scan = make_real_agent() +base = scan.predict_long(LONG, Q) +nwin = sum(scan._forward_calls) // 2 +check("scan/more than one window", nwin > 1, True) +check("scan/the scan reports the windows it read", base["usage"]["windows"], nwin) +check("scan/the deciding window is the last", + base["answers"]["dept"]["window"]["index"], nwin - 1) + +# The instrument for sections 8-8c: a hook that answers must return, and a hook that replaces the +# window list must raise. Letting either unexpected case propagate would abort the suite and hide +# every check after it, so each call is captured and its type asserted by a named check. +def _attempt(fn): + """Call `fn`, returning (value, exception).""" + try: + return fn(), None + except BaseException as exc: # noqa: BLE001 + return None, exc + + +def _kind(exc): + return exc.__class__.__name__ if exc else None -a = make_agent(one_document_answer) +def _narrow(keep): + """A start hook that truncates the window list and lets inference run on what is left.""" + def hook(ctx): + ctx.states = list(ctx.states)[:keep] + return hook + + +a = make_real_agent() with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - res = a.predict_long(LONG, Q) -nwin = len(a._calls["batch_states"]) -check("skip/the state was split into several windows", nwin > 1, True) -check("skip/no window was scored", res["usage"]["windows"], 0) -check("skip/the payload tokens are kept", res["usage"]["input_tokens"], 3) + res, skip_exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.skip([DOC]))) +check("skip/one payload for the document is never refused as a count mismatch", _kind(skip_exc), None) +check("skip/no forward pass ran", a._forward_calls, []) +usage, answers = (res or {}).get("usage", {}), (res or {}).get("answers", {}) +check("skip/no window was scored", usage.get("windows", ""), 0) +check("skip/the payload tokens are kept", usage.get("input_tokens", ""), 3) +check("skip/the hook's answer is returned", answers.get("dept", {}).get("choice"), "b") check("skip/no deciding window is claimed", - sorted(k for v in res["answers"].values() for k in v if k == "window"), []) -check("skip/the hook's answer is returned", res["answers"]["dept"]["choice"], "a") + sorted(k for v in answers.values() for k in v if k == "window"), []) check("skip/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"]) -# 9. a payload that matches neither the document nor the windows is an error, not a guess +# 8b. a start hook that replaces the window list is the other reading of the same count, and it is +# not a hook answer: inference ran on the states the hook left behind. +a = make_real_agent() +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _, narrow_exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(1))) +narrow_err = str(narrow_exc) if narrow_exc else "" +check("narrow/rejects a list shortened to one state", _kind(narrow_exc), "ValueError") +# Why this check sits next to the rejection: the forward is the proof that this was a rewrite and +# not an answer, and it is what made the two cases indistinguishable before. +check("narrow/a forward did run, so nothing was answered before inference", + a._forward_calls, [2]) +check("narrow/the rewrite is not called a hook answer", + [w.category.__name__ for w in caught], []) +check_true("narrow/the error names the split it could not attribute", + ("split into %d windows" % nwin) in narrow_err, narrow_err) +check_true("narrow/the error offers ctx.skip as the way to answer", + "ctx.skip" in narrow_err, narrow_err) +a = make_real_agent() +check_raises("narrow/rejects a list shortened to two states", ValueError, + lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(2))) +# the other direction, so the rule is the count rather than a hook that dropped something. The +# error must name the caller's split: `predict_long` hands the hook a copy, so an in-place +# `append` grows the hook's list and leaves the attributed windows alone. +a = make_real_agent() +_, added_exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.append("invented"))) +added_err = str(added_exc) if added_exc else "" +check("grow/rejects a list with a state added", _kind(added_exc), "ValueError") +check_true("grow/the count is the caller's split, not the hook's list", + ("split into %d windows" % nwin) in added_err, added_err) + +# 8c. rewriting the windows themselves stays supported: the count holds, so the attribution does +a = make_real_agent() +res, exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.extend(()))) +check("rewrite/a no-op on the list scans as usual", _kind(exc), None) +check("rewrite/a no-op on the list keeps the count", + ((res or {}).get("usage") or {}).get("windows", ""), nwin) +a = make_real_agent() +res, exc = _attempt(lambda: a.predict_long( + LONG, Q, on_predict_start=lambda ctx: setattr(ctx, "states", [s.upper() for s in ctx.states]))) +check("rewrite/the hook's text is what was scored", (a._encoded or [""])[0].isupper(), True) +check("rewrite/the same count is the same scan", _kind(exc), None) +check("rewrite/the count is unchanged", ((res or {}).get("usage") or {}).get("windows", ""), nwin) +check("rewrite/the deciding window is still named", + ((res or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1) + +# 8d. the fits-in-one-window path answers to a hook too, so 0 keeps meaning "no window scored this" +a = make_real_agent() +short_hooked, exc = _attempt( + lambda: a.predict_long({"body": "x" * 50}, Q, on_predict_start=lambda ctx: ctx.skip([DOC]))) +check("short skip/no forward pass", a._forward_calls, []) +check("short skip/a hook answer is not counted as a window read", + ((short_hooked or {}).get("usage") or {}).get("windows", ""), 0) +a = make_real_agent() +short_plain, exc = _attempt(lambda: a.predict_long({"body": "x" * 50}, Q)) +check("short skip/the same state without a hook is one window", + ((short_plain or {}).get("usage") or {}).get("windows", ""), 1) + +# 9. a batch call that returns the wrong count for its own states is an error, not a guess a = make_agent(lambda s, q: [{"answers": {}, "usage": {}}] * 2) -check_raises("skip/rejects a count matching nothing", ValueError, lambda: a.predict_long(LONG, Q)) +check_raises("count/rejects a result count matching nothing", ValueError, + lambda: a.predict_long(LONG, Q)) # 10. the count is total, so one question works on every result predict_long can return short_all = make_agent(lambda s, q: []).predict_long({"body": "x" * 50}, Q) @@ -191,12 +345,16 @@ def one_document_answer(states, q): scanned_all = scan_agent.predict_long(LONG, Q) with warnings.catch_warnings(): warnings.simplefilter("ignore") - hook_all = make_agent(one_document_answer).predict_long(LONG, Q) + hook_all, hook_exc = _attempt( + lambda: make_real_agent().predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.skip([DOC]))) +check("total/a hook answer returns rather than being refused", _kind(hook_exc), None) check("total/no path leaves the key out", - [r["usage"].get("windows", "") for r in (short_all, scanned_all, hook_all)], + [(r or {}).get("usage", {}).get("windows", "") + for r in (short_all, scanned_all, hook_all)], [1, len(scan_agent._calls["batch_states"]), 0]) check("total/0 separates a hook answer from a single-window one", - (hook_all["usage"].get("windows", ""), short_all["usage"].get("windows", "")), + ((hook_all or {}).get("usage", {}).get("windows", ""), + short_all["usage"].get("windows", "")), (0, 1)) @@ -208,6 +366,18 @@ def one_document_answer(states, q): check("docs/README states the key is total", "The key is total" in bullet, True) check("docs/README documents all three counts", sorted(set(re.findall(r"`([0-9N])`", bullet))), ["0", "1", "N"]) +# The two rules this branch added, pinned where they are taught rather than in the code comment +hooks_bullet = README[README.index("Hooks wrap the inference that answers the state"):][:800] +check("docs/README says a hook may not change the window count or order", + "reorder windows" in hooks_bullet, True) +check("docs/README names ctx.skip as the way to answer", "ctx.skip(...)" in hooks_bullet, True) +API = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "docs", "hooks", "api.md"), encoding="utf-8").read() +api_para = API[API.index("On `predict_long` the hooks wrap"):][:1200] +check("docs/api.md matches: count and order are fixed", + "or their" in api_para and "number of states raises" in api_para, True) +check("docs/api.md matches: the hook answer reads as zero windows", + 'usage["windows"]` at `0' in api_para, True) check("docs/predict_long's docstring says the key is total", "always present" in (Agent.predict_long.__doc__ or ""), True) From 9ae2318f19f939fe3ae23112daf8e699d63d4227 Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:56:02 +0545 Subject: [PATCH 6/7] fix(agent): aggregate a rewritten scan without naming a window for it The check that read a count of one as a hook answer still used the count as evidence. A start hook may replace `ctx.states` (`docs/hooks/api.md`), so narrowing the scan to one window is a supported outcome and not an answer: a forward ran and booked tokens, and raising for two windows discarded work the caller had already paid for. `predict_long` now asks the observer appended after the caller's own start hooks what reached inference, which makes the three cases total: a `ctx.skip()` payload comes back with no attribution and 0 windows; an unchanged scan aggregates with the deciding window named; a rewritten scan aggregates over the states that were scored, with `usage["windows"]` as their count and no `answer["window"]` -- the token offsets describe the windows this method built, not the text the model read. A count the observer cannot explain (no hook chain ran) stays an error. --- README.md | 11 ++-- docs/hooks/api.md | 21 +++--- laya/agent.py | 84 +++++++++++++++--------- tests/test_predict_long.py | 131 +++++++++++++++++++++---------------- 4 files changed, 148 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index c87cb1f87..1da97ee2c 100644 --- a/README.md +++ b/README.md @@ -581,11 +581,12 @@ result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan, answered the document before the model read any of it. - Hooks wrap the inference that answers the state, so on a scanned document `on_predict_start` fires once with `ctx.states` holding the decoded windows, not the state you passed in (it was - tokenized to produce them). A start hook may rewrite those window texts, and what is scored is - what it leaves behind, but it may not add, drop or reorder windows: the answer attributes itself - to a span of your document by index, so a call that runs inference over any other number of - states raises. A hook that means to answer the document calls `ctx.skip(...)` instead: its result - comes back with no `answer["window"]` and `usage["windows"]` at 0, because no window scored it. + tokenized to produce them). A start hook may replace that list: the answers are aggregated over + whatever reached inference, and `usage["windows"]` counts those states. What a rewritten scan + costs is the attribution — `answer["window"]` names a span of *your* document, so it is only + reported when the scan reached inference unchanged. A hook that means to answer the document + calls `ctx.skip(...)` instead: its result comes back with no `answer["window"]` and + `usage["windows"]` at 0, because no window scored it. A smaller `window` isolates a short deciding span better (it becomes a larger fraction of its window); the default (`max_len - head_max_len`) favors context and throughput. Output shape matches diff --git a/docs/hooks/api.md b/docs/hooks/api.md index 0f5d83d96..fb580bee4 100644 --- a/docs/hooks/api.md +++ b/docs/hooks/api.md @@ -242,15 +242,18 @@ agent.predict(...) # alias of system_one - On `predict_long` the hooks wrap the inference that answers the state, which for a document needing several windows is the single shared `predict_batch` over them: `on_predict_start` fires once, and `ctx.states` holds the decoded window texts in scan order rather than the caller's - state, which was tokenized to produce them. A start hook may rewrite the text of those windows, - and what is scored is what it leaves behind, but it may not change how many there are or their - order -- the answer attributes itself to a span of the caller's document by index, so a call that - runs inference over any other number of states raises. A hook that means to answer the document - calls `ctx.skip(...)` instead: its one result is returned as-is with no `window` key and - `usage["windows"]` at `0` -- no window scored it. `usage["windows"]` is total over the three - paths: `0` for a hook answer on either path, so a cached answer never reads as a window the model - read, `1` for a state that fit a single window and was scored, and `N` for a document scanned in - `N` overlapping windows. + state, which was tokenized to produce them. `states` is mutable from a start hook, so the scan + that reaches inference need not be the split `predict_long` computed. What changes is what the + answer can claim: + + | what the start hook did | `usage["windows"]` | `answer["window"]` | + |---|---|---| + | answered with `ctx.skip([result])` | `0` | absent -- no window scored it | + | left the scan as it was built | `N` | present -- `index`, `token_start`/`token_end` name the deciding span | + | replaced the scan, in any way | the states that were scored | absent -- the offsets describe `predict_long`'s windows, not the text that was scored | + + `usage["windows"]` is total over every path, including the one where a state already fit a single + window, so a cached answer never reads as a window the model read. ### Router diff --git a/laya/agent.py b/laya/agent.py index c026f589d..66e7e60c6 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -217,22 +217,28 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype: def _start_evidence(): - """A recorder for `predict_long`: what the start-hook chain did, rather than what it cost. + """A recorder for `predict_long`: what the start-hook chain left for inference to run on. - Why a recorder at all: the result count cannot tell a hook that answered the document from a - hook that replaced the window list. Both return fewer results than there were windows, and the - two need opposite readings -- the first scored no window, the second scored the ones the hook - left behind, and booked tokens for them. `ctx.results` already distinguishes them, and only the - engine's own context sees it, so `predict_long` appends this probe to the call's start hooks. + Why a recorder at all: `ctx.skip()` only assigns `ctx.results` (see `laya.hooks.PredictContext`), + so outside `predict_batch` the only account of what the hooks did is the context they were + handed. The result count cannot tell a hook that answered the document from one that replaced + the window list -- both return fewer results than there were windows, and the two need opposite + readings: the first scored no window, the second scored the ones it left behind and booked + tokens for them. + + The probe belongs last in the chain because `compose_hooks` orders defaults, then installed, + then per-call hooks, and `dispatch` calls them in order, so a probe appended after the caller's + own start hooks observes exactly what `predict_batch` is about to act on. Returns the probe and the dict it fills: `answered` is whether a hook replaced the call before - inference, `states` is the number of states that reached it. + inference, `states` is a snapshot of the states that reached it (`None` if the probe never ran, + which means `predict_batch` was replaced and no hook chain was dispatched). """ evidence = {"answered": False, "states": None} def probe(ctx): evidence["answered"] = ctx.results is not None - evidence["states"] = len(ctx.states) + evidence["states"] = list(ctx.states) return probe, evidence @@ -1014,11 +1020,15 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ The hooks wrap the inference that answers the state, which for a document needing several windows is the one shared `predict_batch` over them: `on_predict_start` fires once, and `ctx.states` holds the decoded window texts in scan order -- not the caller's `state`, which - was tokenized to produce them. A start hook may rewrite the text of those windows, and what - is scored is what it leaves behind. It may not change how many there are, or their order: - the per-window attribution names spans of the caller's document by index, so a different - count raises. A hook that means to answer the document itself calls `ctx.skip(...)`, whose - single payload is returned without window attribution. + was tokenized to produce them. Three outcomes follow from what the chain leaves behind: + + * `ctx.skip([result])` answers the document: the payload comes back with no window + attribution and `usage["windows"]` at 0, because nothing was scored + * a scan left as this method built it: every window is scored, each answer carries + `answer["window"]`, and `usage["windows"]` is the window count + * a rewritten scan (`ctx.states` replaced, in any way): the answers are aggregated over + the states that were scored, but no answer carries `answer["window"]` -- the offsets + above describe this method's windows, not the text the model read Args: window: state tokens per window. Defaults to the per-question state budget @@ -1042,9 +1052,10 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ Returns a single result dict, the same shape as `system_one`, with `usage["windows"]` added. The key is always present and counts the windows the model scored to produce the answer: `1` - for a state that fit one window, `N` for a document scanned in `N` overlapping windows, and - `0` when a start hook answered before any window was scored -- on either path, so a cached - answer never reads as a window the model read. + for a state that fit one window, `N` for a document scanned in `N` overlapping windows (or + the `N` a start hook rewrote them to), and `0` when a start hook answered the document, or + left no states to score, before any window was read -- on either path, so a cached answer + never reads as a window the model read. """ if aggregate != "auto": raise ValueError("predict_long: only aggregate='auto' is supported") @@ -1109,18 +1120,30 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ document["usage"]["windows"] = 0 return document - if len(results) != len(windows): - # A hook can also rewrite `ctx.states` and let inference run on what it left behind, - # which moves the count without answering anything. Those results cannot be attributed - # to windows of the caller's document -- the states that produced them are the hook's - # -- so refuse rather than name a span that was never read. - survived = ("" if evidence["states"] is None else - " (%d states reached inference)" % evidence["states"]) + # The contract on `ctx.states` is that a start hook may replace it (see `docs/hooks/api.md`), + # so a scan that comes back different from the split above is a supported outcome, not a + # failure to report: the states that were scored are the hook's, while `starts` describes + # this method's windows. Aggregate what came back and name nothing. + rewritten = evidence["states"] is not None and evidence["states"] != windows + + if len(results) != len(windows) and not rewritten: + # The observer saw the scan leave the hook chain and it is the one computed above, so + # nothing here explains a count that is not the split. That is `predict_batch` + # disagreeing with its own input -- a bug, or a replacement that never dispatched hooks. + seen = ("the scan that reached inference was the split made here" + if evidence["states"] is not None + else "no start hook chain ran, so nothing rewrote the scan") raise ValueError( "predict_long: the state was split into %d windows and the call returned %d" - " results%s; a start hook may rewrite the text of a window but not their number" - " or their order -- call ctx.skip([result]) to answer the document without scoring" - " it" % (len(windows), len(results), survived)) + " results, and %s" % (len(windows), len(results), seen)) + + if not results: + # A hook that left no states scored nothing, which is what 0 already means here; the + # questions go unanswered rather than being aggregated over an empty list. + warnings.warn("laya: predict_long: a start hook left no states to score, so the call" + " aggregated nothing and returns no answers", RuntimeWarning, stacklevel=2) + return {"model": "laya-rl-agent", "answers": {}, + "usage": {**aggregate_usage(results), "windows": 0}} ids = list(questions.keys()) internal = {qid: self._to_internal(questions[qid]) for qid in ids} @@ -1139,9 +1162,10 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ ans = per[best] # Name the window that decided, so a caller can check the deciding span itself. The # probability here is that window's, NOT a document-level calibrated number. - ans["window"] = {"index": best, "token_start": starts[best], - "token_end": min(starts[best] + budget, len(state_ids)), - "count": len(windows)} + if not rewritten: + ans["window"] = {"index": best, "token_start": starts[best], + "token_end": min(starts[best] + budget, len(state_ids)), + "count": len(results)} answers[qid] = ans # Aggregate usage generically so fields predict_batch may grow later (e.g. the fallback # counters from #351) are propagated, not silently dropped: sum numeric fields across @@ -1151,7 +1175,7 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ for key, val in r["usage"].items(): usage[key] = (usage.get(key, 0) + val) if isinstance(val, (int, float)) else val usage["output_tokens"] = 0 - usage["windows"] = len(windows) + usage["windows"] = len(results) return {"model": "laya-rl-agent", "answers": answers, "usage": usage} @torch.no_grad() diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index ab4471ddc..2dfce4089 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -227,19 +227,10 @@ def sentinel_start(ctx): "flag": {"type": "noul", "noul": 0.2, "answer_confidence": 0.8}}, "usage": {"input_tokens": 3, "output_tokens": 0}} -# The control first: how many windows this state really splits into, measured from the forward -# passes it costs rather than from the key under test. Two questions, one row each per window. -scan = make_real_agent() -base = scan.predict_long(LONG, Q) -nwin = sum(scan._forward_calls) // 2 -check("scan/more than one window", nwin > 1, True) -check("scan/the scan reports the windows it read", base["usage"]["windows"], nwin) -check("scan/the deciding window is the last", - base["answers"]["dept"]["window"]["index"], nwin - 1) - -# The instrument for sections 8-8c: a hook that answers must return, and a hook that replaces the -# window list must raise. Letting either unexpected case propagate would abort the suite and hide -# every check after it, so each call is captured and its type asserted by a named check. +# The instrument for sections 8-8e: a predict_long call is captured rather than let propagate, so +# a case that ends in the wrong shape -- an answer that should have come back raising, or a +# rejection that should not have happened at all -- fails a named check instead of aborting the run +# and hiding every check after it. def _attempt(fn): """Call `fn`, returning (value, exception).""" try: @@ -252,6 +243,19 @@ def _kind(exc): return exc.__class__.__name__ if exc else None +# The control first: how many windows this state really splits into, measured from the forward +# passes it costs rather than from the key under test. Two questions, one row each per window. +scan = make_real_agent() +base, base_exc = _attempt(lambda: scan.predict_long(LONG, Q)) +check("scan/an unhooked scan returns", _kind(base_exc), None) +nwin = sum(scan._forward_calls) // 2 +check("scan/more than one window", nwin > 1, True) +check("scan/the scan reports the windows it read", + ((base or {}).get("usage") or {}).get("windows", ""), nwin) +check("scan/the deciding window is the last", + ((base or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1) + + def _narrow(keep): """A start hook that truncates the window list and lets inference run on what is left.""" def hook(ctx): @@ -274,53 +278,70 @@ def hook(ctx): sorted(k for v in answers.values() for k in v if k == "window"), []) check("skip/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"]) -# 8b. a start hook that replaces the window list is the other reading of the same count, and it is -# not a hook answer: inference ran on the states the hook left behind. +# 8b. a start hook that replaces the scan is the middle row: inference ran on the hook's states, so +# the answers aggregate, the count is theirs, and no span of the caller's document is named -- the +# offsets computed above describe windows that were not scored. a = make_real_agent() with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - _, narrow_exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(1))) -narrow_err = str(narrow_exc) if narrow_exc else "" -check("narrow/rejects a list shortened to one state", _kind(narrow_exc), "ValueError") -# Why this check sits next to the rejection: the forward is the proof that this was a rewrite and -# not an answer, and it is what made the two cases indistinguishable before. -check("narrow/a forward did run, so nothing was answered before inference", - a._forward_calls, [2]) -check("narrow/the rewrite is not called a hook answer", + narrow, narrow_exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(1))) +check("narrow/one state left behind is a scan, not a rejected count", _kind(narrow_exc), None) +# Why the forward sits next to the count: it is the proof that tokens were spent, which is exactly +# what made this case read as a hook answer before. +check("narrow/inference ran on the state that was left", a._forward_calls, [2]) +check("narrow/the count is the state scored, not the split made", + ((narrow or {}).get("usage") or {}).get("windows", ""), 1) +check("narrow/the tokens that forward booked are kept", + ((narrow or {}).get("usage") or {}).get("input_tokens", 0) > 0, True) +check("narrow/no span of the caller's document is named", + sorted(k for v in (narrow or {}).get("answers", {}).values() for k in v if k == "window"), []) +check("narrow/the answer is still aggregated", + (narrow or {}).get("answers", {}).get("dept", {}).get("type"), "choice") +check("narrow/a rewrite is not warned about as a hook answer", [w.category.__name__ for w in caught], []) -check_true("narrow/the error names the split it could not attribute", - ("split into %d windows" % nwin) in narrow_err, narrow_err) -check_true("narrow/the error offers ctx.skip as the way to answer", - "ctx.skip" in narrow_err, narrow_err) a = make_real_agent() -check_raises("narrow/rejects a list shortened to two states", ValueError, - lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(2))) -# the other direction, so the rule is the count rather than a hook that dropped something. The -# error must name the caller's split: `predict_long` hands the hook a copy, so an in-place -# `append` grows the hook's list and leaves the attributed windows alone. +two, exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(2))) +check("narrow/two states left behind are counted as two", + ((two or {}).get("usage") or {}).get("windows", ""), 2) +check("narrow/and two states is what the forward saw", a._forward_calls, [4]) +# the other direction, and in place: `predict_long` hands the hook a copy of its split, so the +# state the hook invented is scored and counted without moving the windows computed above. a = make_real_agent() -_, added_exc = _attempt( +grown, exc = _attempt( lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.append("invented"))) -added_err = str(added_exc) if added_exc else "" -check("grow/rejects a list with a state added", _kind(added_exc), "ValueError") -check_true("grow/the count is the caller's split, not the hook's list", - ("split into %d windows" % nwin) in added_err, added_err) +check("grow/the hook's added state is scored", + ((grown or {}).get("usage") or {}).get("windows", ""), nwin + 1) +check("grow/the caller's split is the one that was extended", a._forward_calls, [2 * (nwin + 1)]) +check("grow/and no span is named", + sorted(k for v in (grown or {}).get("answers", {}).values() for k in v if k == "window"), []) -# 8c. rewriting the windows themselves stays supported: the count holds, so the attribution does +# 8c. an unchanged scan keeps the attribution; any rewrite drops it, even one of the same length a = make_real_agent() -res, exc = _attempt( +noop, exc = _attempt( lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.extend(()))) -check("rewrite/a no-op on the list scans as usual", _kind(exc), None) -check("rewrite/a no-op on the list keeps the count", - ((res or {}).get("usage") or {}).get("windows", ""), nwin) +check("rewrite/a no-op on the list changes nothing", _kind(exc), None) +check("rewrite/a no-op keeps the count", ((noop or {}).get("usage") or {}).get("windows", ""), nwin) +check("rewrite/a no-op keeps the deciding window", + ((noop or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1) a = make_real_agent() -res, exc = _attempt(lambda: a.predict_long( +upper, exc = _attempt(lambda: a.predict_long( LONG, Q, on_predict_start=lambda ctx: setattr(ctx, "states", [s.upper() for s in ctx.states]))) check("rewrite/the hook's text is what was scored", (a._encoded or [""])[0].isupper(), True) -check("rewrite/the same count is the same scan", _kind(exc), None) -check("rewrite/the count is unchanged", ((res or {}).get("usage") or {}).get("windows", ""), nwin) -check("rewrite/the deciding window is still named", - ((res or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1) +check("rewrite/the same length is still the same count", + ((upper or {}).get("usage") or {}).get("windows", ""), nwin) +check("rewrite/a text rewrite names no span", + sorted(k for v in (upper or {}).get("answers", {}).values() for k in v if k == "window"), []) + +# 8e. a hook that leaves nothing scores nothing: 0 windows and no answers, not a max() over [] +a = make_real_agent() +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + empty, exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: setattr(ctx, "states", []))) +check("empty/nothing scored returns instead of raising", _kind(exc), None) +check("empty/no answer is invented", (empty or {"answers": "?"}).get("answers", "?"), {}) +check("empty/the count says so", ((empty or {}).get("usage") or {}).get("windows", ""), 0) +check("empty/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"]) # 8d. the fits-in-one-window path answers to a hook too, so 0 keeps meaning "no window scored this" a = make_real_agent() @@ -367,17 +388,17 @@ def hook(ctx): check("docs/README documents all three counts", sorted(set(re.findall(r"`([0-9N])`", bullet))), ["0", "1", "N"]) # The two rules this branch added, pinned where they are taught rather than in the code comment -hooks_bullet = README[README.index("Hooks wrap the inference that answers the state"):][:800] -check("docs/README says a hook may not change the window count or order", - "reorder windows" in hooks_bullet, True) +hooks_bullet = README[README.index("Hooks wrap the inference that answers the state"):][:900] +check("docs/README says attribution survives only an unchanged scan", + "reported when the scan reached inference unchanged" in hooks_bullet, True) check("docs/README names ctx.skip as the way to answer", "ctx.skip(...)" in hooks_bullet, True) API = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "docs", "hooks", "api.md"), encoding="utf-8").read() -api_para = API[API.index("On `predict_long` the hooks wrap"):][:1200] -check("docs/api.md matches: count and order are fixed", - "or their" in api_para and "number of states raises" in api_para, True) -check("docs/api.md matches: the hook answer reads as zero windows", - 'usage["windows"]` at `0' in api_para, True) +api_para = API[API.index("On `predict_long` the hooks wrap"):][:1600] +check("docs/api.md tabulates the rewritten scan as its own case", + "| replaced the scan, in any way |" in api_para, True) +check("docs/api.md tabulates a hook answer as zero windows", + "| answered with `ctx.skip([result])` | `0` |" in api_para, True) check("docs/predict_long's docstring says the key is total", "always present" in (Agent.predict_long.__doc__ or ""), True) From c12f7013cc451d5fab002e673c3eb8146efd31b6 Mon Sep 17 00:00:00 2001 From: Aashish <145881415+aashish254@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:57:53 +0545 Subject: [PATCH 7/7] test(agent): name the exception a rewritten scan raises, not just the count it reports The aliasing bug is visible only as a type: without the copy of the split, `ctx.states.append()` grows both lists, the counts agree, and the attribution indexes a shorter `starts`. So the check asserts the call returns at all before it asserts what it returned. --- tests/test_predict_long.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index 2dfce4089..8325d2c2c 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -309,6 +309,7 @@ def hook(ctx): a = make_real_agent() grown, exc = _attempt( lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.append("invented"))) +check("grow/scoring one invented state is not an error", _kind(exc), None) check("grow/the hook's added state is scored", ((grown or {}).get("usage") or {}).get("windows", ""), nwin + 1) check("grow/the caller's split is the one that was extended", a._forward_calls, [2 * (nwin + 1)])