diff --git a/README.md b/README.md index 54c20cf9a..1da97ee2c 100644 --- a/README.md +++ b/README.md @@ -570,12 +570,23 @@ per question: ```python result = agent.predict_long(state, questions) # windows the state, one result back result = agent.predict_long(state, questions, window=256) # smaller window isolates a localized span +result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan, instrumented ``` - `noul` takes the strongest window (the statement holds if any window supports it). - `choice` / `score` take the most-confident window — averaging over a long, mostly-neutral document lets the neutral majority out-vote the one window that saw the deciding span. -- A state that already fits one window is passed straight to `system_one` (identical output). +- A state that already fits one window is passed straight to `system_one` (identical output, plus + `usage["windows"] = 1`). The key is total: `1` single window, `N` scanned windows, `0` a hook + answered the document before the model read any of it. +- Hooks wrap the inference that answers the state, so on a scanned document `on_predict_start` + fires once with `ctx.states` holding the decoded windows, not the state you passed in (it was + tokenized to produce them). A start hook may replace that list: the answers are aggregated over + whatever reached inference, and `usage["windows"]` counts those states. What a rewritten scan + costs is the attribution — `answer["window"]` names a span of *your* document, so it is only + reported when the scan reached inference unchanged. A hook that means to answer the document + calls `ctx.skip(...)` instead: its result comes back with no `answer["window"]` and + `usage["windows"]` at 0, because no window scored it. A smaller `window` isolates a short deciding span better (it becomes a larger fraction of its window); the default (`max_len - head_max_len`) favors context and throughput. Output shape matches diff --git a/docs/hooks/api.md b/docs/hooks/api.md index 8817e6663..fb580bee4 100644 --- a/docs/hooks/api.md +++ b/docs/hooks/api.md @@ -229,11 +229,31 @@ agent.system_one(state, questions, hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None, hooks_timeout=None, max_len=None, head_max_len=None) +agent.predict_long(state, questions, window=None, stride=None, aggregate="auto", + batch_size=None, lang=None, + hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None, + hooks_timeout=None) + agent.predict(...) # alias of system_one ``` - `hooks_raise` and `hooks_timeout` on a per-call method default to `None`, meaning "use the instance value". - `hooks_concurrent` is instance-level only. +- On `predict_long` the hooks wrap the inference that answers the state, which for a document + needing several windows is the single shared `predict_batch` over them: `on_predict_start` fires + once, and `ctx.states` holds the decoded window texts in scan order rather than the caller's + state, which was tokenized to produce them. `states` is mutable from a start hook, so the scan + that reaches inference need not be the split `predict_long` computed. What changes is what the + answer can claim: + + | what the start hook did | `usage["windows"]` | `answer["window"]` | + |---|---|---| + | answered with `ctx.skip([result])` | `0` | absent -- no window scored it | + | left the scan as it was built | `N` | present -- `index`, `token_start`/`token_end` name the deciding span | + | replaced the scan, in any way | the states that were scored | absent -- the offsets describe `predict_long`'s windows, not the text that was scored | + + `usage["windows"]` is total over every path, including the one where a state already fit a single + window, so a cached answer never reads as a window the model read. ### Router diff --git a/laya/agent.py b/laya/agent.py index 1b58de6f3..66e7e60c6 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -29,8 +29,8 @@ temp_bucket, ) from .hooks import ( - HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks, - validate_timeout, + HookRegistry, PredictContext, _as_sequence, aggregate_usage, compose_hooks, dispatch, + normalise_hooks, validate_timeout, ) from .revisions import resolve_revision, snapshot_revision, verify_digests @@ -216,6 +216,40 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype: return amp_dtype(checkpoint_default) +def _start_evidence(): + """A recorder for `predict_long`: what the start-hook chain left for inference to run on. + + Why a recorder at all: `ctx.skip()` only assigns `ctx.results` (see `laya.hooks.PredictContext`), + so outside `predict_batch` the only account of what the hooks did is the context they were + handed. The result count cannot tell a hook that answered the document from one that replaced + the window list -- both return fewer results than there were windows, and the two need opposite + readings: the first scored no window, the second scored the ones it left behind and booked + tokens for them. + + The probe belongs last in the chain because `compose_hooks` orders defaults, then installed, + then per-call hooks, and `dispatch` calls them in order, so a probe appended after the caller's + own start hooks observes exactly what `predict_batch` is about to act on. + + Returns the probe and the dict it fills: `answered` is whether a hook replaced the call before + inference, `states` is a snapshot of the states that reached it (`None` if the probe never ran, + which means `predict_batch` was replaced and no hook chain was dispatched). + """ + evidence = {"answered": False, "states": None} + + def probe(ctx): + evidence["answered"] = ctx.results is not None + evidence["states"] = list(ctx.states) + + return probe, evidence + + +def _with_start_probe(hook_kwargs, probe): + """`hook_kwargs` with `probe` appended after the caller's own start hooks.""" + kwargs = dict(hook_kwargs) + kwargs["on_predict_start"] = list(_as_sequence(hook_kwargs.get("on_predict_start"))) + [probe] + return kwargs + + class Agent(HookRegistry): """System 1 decision model runtime: fast, non-autoregressive, calibrated decisions.""" @@ -956,7 +990,10 @@ def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[st def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]], window: Optional[int] = None, stride: Optional[int] = None, aggregate: str = "auto", batch_size: Optional[int] = None, - lang: Optional[str] = None) -> Dict[str, Any]: + lang: Optional[str] = None, + hooks=None, on_predict_start=None, on_predict_end=None, + hooks_raise: Optional[bool] = None, + hooks_timeout: Optional[float] = None) -> Dict[str, Any]: """Evaluate questions over a state longer than the context window, scanning it in overlapping windows and aggregating per question. @@ -980,6 +1017,19 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ A state that already fits one window is passed straight to `system_one` (identical output). + The hooks wrap the inference that answers the state, which for a document needing several + windows is the one shared `predict_batch` over them: `on_predict_start` fires once, and + `ctx.states` holds the decoded window texts in scan order -- not the caller's `state`, which + was tokenized to produce them. Three outcomes follow from what the chain leaves behind: + + * `ctx.skip([result])` answers the document: the payload comes back with no window + attribution and `usage["windows"]` at 0, because nothing was scored + * a scan left as this method built it: every window is scored, each answer carries + `answer["window"]`, and `usage["windows"]` is the window count + * a rewritten scan (`ctx.states` replaced, in any way): the answers are aggregated over + the states that were scored, but no answer carries `answer["window"]` -- the offsets + above describe this method's windows, not the text the model read + Args: window: state tokens per window. Defaults to the per-question state budget (`max_len - head_max_len - 8`) -- the most a window can hold for every question. @@ -993,20 +1043,45 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ aggregate: "auto" (the per-type rules above) is the only mode for now. batch_size: cap on windows per forward pass, to bound memory on very long states. lang: per-language temperature selection, as in `system_one`. + hooks (HookArg): Per-call hooks, appended after any installed on the Agent. + See `laya.hooks`. + on_predict_start (PredictHookArg): A per-call start hook, as in `system_one`. + on_predict_end (PredictHookArg): A per-call end hook, as in `system_one`. + hooks_raise: Override the Agent's `hooks_raise` for this call. + hooks_timeout: Override the Agent's `hooks_timeout` for this call. Returns a single result dict, the same shape as `system_one`, with `usage["windows"]` added. + The key is always present and counts the windows the model scored to produce the answer: `1` + for a state that fit one window, `N` for a document scanned in `N` overlapping windows (or + the `N` a start hook rewrote them to), and `0` when a start hook answered the document, or + left no states to score, before any window was read -- on either path, so a cached answer + never reads as a window the model read. """ if aggregate != "auto": raise ValueError("predict_long: only aggregate='auto' is supported") + hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start, + "on_predict_end": on_predict_end, "hooks_raise": hooks_raise, + "hooks_timeout": hooks_timeout} max_len = self.cfg.get("max_len", 512) head_max_len = self.cfg.get("head_max_len", 192) budget = window if (window and window > 0) else max(64, max_len - head_max_len - 8) state_ids = self.tok(serialize_state(state).replace(self.tok.mask_token, " "), add_special_tokens=False)["input_ids"] - # Fits in one window: identical to a plain call, no windowing overhead. + # Fits in one window: identical to a plain call, no windowing overhead. `windows` is still + # written, so the key is total over the three paths this method can take and a caller can + # ask "how much of the document did the model read?" without handling a KeyError on the + # shortest, most common inputs. The result is copied first: a start hook that answers with + # `ctx.skip(...)` hands back its own payload dict, and it may be a cached object. if len(state_ids) <= budget: - return self.system_one(state, questions, lang=lang) + probe, evidence = _start_evidence() + single = dict(self.system_one(state, questions, lang=lang, + **_with_start_probe(hook_kwargs, probe))) + # Why 0 for a hook answer here: the state did fit one window, but no window was scored, + # which is the same fact the multi-window path reports as 0. Reading 1 would make a + # cached answer and a served answer agree on how much of the input the model saw. + single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1} + return single step = stride if (stride and stride > 0) else max(1, budget // 2) windows, starts = [], [] @@ -1021,7 +1096,54 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ break i += step - results = self.predict_batch(windows, questions, batch_size=batch_size, lang=lang) + probe, evidence = _start_evidence() + # `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that + # mutates it in place (`append`, `sort`) would otherwise also grow the split this call + # attributes answers to, and the counts would agree while `starts` no longer lined up. + results = self.predict_batch(list(windows), questions, batch_size=batch_size, lang=lang, + **_with_start_probe(hook_kwargs, probe)) + + if evidence["answered"]: + # The hook replaced the call before any window was scored. Aggregating over its payload + # would pick between answers that were never scored and name a deciding window that + # decided nothing, so pass the document answer through unattributed and report the + # count as what it is: none of them. + if len(results) != 1: + raise ValueError( + "predict_long: a start hook answered this state with %d results; ctx.skip()" + " takes one result for the document, not one per window" % len(results)) + warnings.warn("laya: predict_long: a hook answered the state before it was scanned, so " + "no window decided the result and none is reported", RuntimeWarning, + stacklevel=2) + document = dict(results[0]) + document["usage"] = dict(document.get("usage") or {}) + document["usage"]["windows"] = 0 + return document + + # The contract on `ctx.states` is that a start hook may replace it (see `docs/hooks/api.md`), + # so a scan that comes back different from the split above is a supported outcome, not a + # failure to report: the states that were scored are the hook's, while `starts` describes + # this method's windows. Aggregate what came back and name nothing. + rewritten = evidence["states"] is not None and evidence["states"] != windows + + if len(results) != len(windows) and not rewritten: + # The observer saw the scan leave the hook chain and it is the one computed above, so + # nothing here explains a count that is not the split. That is `predict_batch` + # disagreeing with its own input -- a bug, or a replacement that never dispatched hooks. + seen = ("the scan that reached inference was the split made here" + if evidence["states"] is not None + else "no start hook chain ran, so nothing rewrote the scan") + raise ValueError( + "predict_long: the state was split into %d windows and the call returned %d" + " results, and %s" % (len(windows), len(results), seen)) + + if not results: + # A hook that left no states scored nothing, which is what 0 already means here; the + # questions go unanswered rather than being aggregated over an empty list. + warnings.warn("laya: predict_long: a start hook left no states to score, so the call" + " aggregated nothing and returns no answers", RuntimeWarning, stacklevel=2) + return {"model": "laya-rl-agent", "answers": {}, + "usage": {**aggregate_usage(results), "windows": 0}} ids = list(questions.keys()) internal = {qid: self._to_internal(questions[qid]) for qid in ids} @@ -1040,9 +1162,10 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ ans = per[best] # Name the window that decided, so a caller can check the deciding span itself. The # probability here is that window's, NOT a document-level calibrated number. - ans["window"] = {"index": best, "token_start": starts[best], - "token_end": min(starts[best] + budget, len(state_ids)), - "count": len(windows)} + if not rewritten: + ans["window"] = {"index": best, "token_start": starts[best], + "token_end": min(starts[best] + budget, len(state_ids)), + "count": len(results)} answers[qid] = ans # Aggregate usage generically so fields predict_batch may grow later (e.g. the fallback # counters from #351) are propagated, not silently dropped: sum numeric fields across @@ -1052,7 +1175,7 @@ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[ for key, val in r["usage"].items(): usage[key] = (usage.get(key, 0) + val) if isinstance(val, (int, float)) else val usage["output_tokens"] = 0 - usage["windows"] = len(windows) + usage["windows"] = len(results) return {"model": "laya-rl-agent", "answers": answers, "usage": usage} @torch.no_grad() diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py index fa750e30b..7dc6422eb 100644 --- a/tests/test_hooks_api.py +++ b/tests/test_hooks_api.py @@ -72,6 +72,7 @@ def check_param(name, fn, param, default, kind=None): # --------------------------------------------------------------- predict surfaces for label, fn in (("Agent.predict_batch", Agent.predict_batch), ("Agent.system_one", Agent.system_one), + ("Agent.predict_long", Agent.predict_long), ("Router.predict", Router.predict), ("ONNXAgent.system_one", ONNXAgent.system_one)): check_param(label, fn, "hooks", None) @@ -80,6 +81,13 @@ def check_param(name, fn, param, default, kind=None): check_param(label, fn, "hooks_raise", None) check_param(label, fn, "hooks_timeout", None) +# The scan sizes its windows from the checkpoint budget, so it takes `window`/`stride` instead +# of the per-call token overrides the single-window entries take. +check_param("Agent.predict_long", Agent.predict_long, "window", None) +check_param("Agent.predict_long", Agent.predict_long, "stride", None) +for param in ("max_len", "head_max_len"): + check("Agent.predict_long/%s not accepted" % param, param in sig(Agent.predict_long), False) + check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None) # per-call token-budget overrides diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py index e8d70688f..8325d2c2c 100644 --- a/tests/test_predict_long.py +++ b/tests/test_predict_long.py @@ -2,14 +2,20 @@ Weight-free. The real forward path is stubbed (predict_batch returns canned per-window answers), so this checks only predict_long's own logic: the fits-in-one-window short-circuit, the overlapping -window split, and the per-type aggregation (noul = strongest window, choice/score = most-confident -window). Numerical behaviour on real weights is exercised in tests/test_local_e2e.py. +window split, the per-type aggregation (noul = strongest window, choice/score = most-confident +window), and the per-call hook controls it forwards to whichever of those two calls runs. Numerical +behaviour on real weights is exercised in tests/test_local_e2e.py. """ import os +import re import sys +import threading +import warnings sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import numpy as np # noqa: E402 + from laya.agent import Agent # noqa: E402 PASS, FAIL = [], [] @@ -30,6 +36,10 @@ def check_raises(name, exc, fn): FAIL.append("%s: did not raise %s" % (name, exc.__name__)) +def check_true(name, cond, detail=""): + (PASS if cond else FAIL).append(name if cond else "%s: %s" % (name, detail)) + + class _Tok: mask_token = "[M]" @@ -46,14 +56,17 @@ def make_agent(batch_result_fn): a.cfg = {"max_len": 100, "head_max_len": 20} # budget = max(64, 100-20-8) = 72 a.tok = _Tok() a._to_internal = staticmethod(Agent._to_internal).__func__ - a._calls = {"system_one": 0, "batch_states": None} + a._calls = {"system_one": 0, "batch_states": None, "system_one_kwargs": None, + "batch_kwargs": None} - def _system_one(state, questions, lang=None): + def _system_one(state, questions, lang=None, **controls): a._calls["system_one"] += 1 + a._calls["system_one_kwargs"] = controls return {"model": "laya-rl-agent", "answers": {"_via": "system_one"}, "usage": {"input_tokens": 1}} - def _predict_batch(states, questions, batch_size=None, lang=None): + def _predict_batch(states, questions, batch_size=None, lang=None, **controls): a._calls["batch_states"] = list(states) + a._calls["batch_kwargs"] = controls return batch_result_fn(list(states), questions) a.system_one = _system_one @@ -61,6 +74,49 @@ def _predict_batch(states, questions, batch_size=None, lang=None): return a +def make_real_agent(): + """`predict_long` on the real `predict_batch`/`system_one`, with only the three composed + helpers stubbed -- the harness `tests/test_hooks.py` uses, so no weights are involved. + + Why both agents: the stubbed `predict_batch` above is the right instrument for what + `predict_long` *forwards*, but it never runs a hook, so anything it says about a hook answering + rests on a result count alone. Here the hook chain really runs and every forward pass is + counted, which is the difference between "a hook answered the document" and "a hook replaced + the window list" -- both change the count. + """ + a = Agent.__new__(Agent) + a.cfg = {"max_len": 100, "head_max_len": 20} # budget = max(64, 100-20-8) = 72 + a.tok = type("Tok", (_Tok,), {"pad_token_id": 0})() + a._to_internal = staticmethod(Agent._to_internal).__func__ + a.model_id = "convaiinnovations/laya" + a.hooks = [] + a.hooks_raise = True + a.hooks_timeout = None + a._hooks_lock = threading.Lock() + a._forward_calls, a._encoded = [], [] + + def _encode_state(state, ids, internal, **overrides): + a._encoded.append(state) + return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids] + + def _forward(b): + n = b["input_ids"].shape[0] + a._forward_calls.append(n) + return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32) + + def _decode_answers(logits, act, items, ids, internal, row, **kw): + conf = 0.4 + 0.05 * row # so the last window decides + return {"dept": {"type": "choice", "choice": "a", "confidence": conf, + "answer_confidence": conf}, + "flag": {"type": "noul", "noul": conf, "confidence": conf, + "answer_confidence": conf}} + + a._encode_state = _encode_state + a._forward = _forward + a._decode_answers = _decode_answers + return a + + Q = {"dept": {"type": "choice", "instructions": "?", "criteria": {"a": "x", "b": "y"}}, "flag": {"type": "noul", "instructions": "?"}} @@ -69,6 +125,17 @@ def _predict_batch(states, questions, batch_size=None, lang=None): short = a.predict_long({"body": "x" * 50}, Q) # 50 tokens <= budget 72 check("short/delegates to system_one", short["answers"], {"_via": "system_one"}) check("short/no predict_batch call", a._calls["batch_states"], None) +check("short/one window is reported", short["usage"].get("windows", ""), 1) +check("short/system_one's own usage is kept", short["usage"].get("input_tokens", ""), 1) + +# 1b. the count is added to a copy, because a start hook that skips hands back the caller's own +# payload dict and that object may be cached and reused +payload = {"model": "laya-rl-agent", "answers": {"_via": "system_one"}, "usage": {"input_tokens": 1}} +a = make_agent(lambda s, q: []) +a.system_one = lambda state, questions, lang=None, **controls: payload +via_hook = a.predict_long({"body": "x" * 50}, Q) +check("short/a new result dict comes back", via_hook is payload, False) +check("short/the caller's payload is not written to", payload["usage"], {"input_tokens": 1}) # 2. long state -> overlapping windows, aggregated per question def canned(states, q): @@ -108,6 +175,234 @@ def canned(states, q): check_raises("aggregate/rejects unknown mode", ValueError, lambda: a.predict_long({"body": "y" * 300}, Q, aggregate="mean")) +# 4. the per-call hook controls `predict`/`system_one`/`predict_batch` take reach the scan +HOOK_KEYS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise", "hooks_timeout") +LONG = {"body": "y" * 300} +sentinel = object() + + +def sentinel_start(ctx): + pass + + +a = make_agent(canned) +res = a.predict_long(LONG, Q, hooks=[sentinel], on_predict_start=sentinel_start, + hooks_raise=False, hooks_timeout=2.5) +got = a._calls["batch_kwargs"] +check("hooks/forwarded to the scan", sorted(got), sorted(HOOK_KEYS)) +check("hooks/the hook list reaches it", got.get("hooks"), [sentinel]) +check("hooks/hooks_raise reaches it", got.get("hooks_raise"), False) +check("hooks/hooks_timeout reaches it", got.get("hooks_timeout"), 2.5) +started = got.get("on_predict_start") +started = started if isinstance(started, list) else ([] if started is None else [started]) +check("hooks/the caller's start hook still runs first", + bool(started) and started[0] is sentinel_start, True) +check("hooks/end stays absent when the caller sets none", got.get("on_predict_end"), None) + +# 5. the same controls reach the fits-in-one-window path +a = make_agent(lambda s, q: []) +a.predict_long({"body": "x" * 50}, Q, hooks=[sentinel], hooks_timeout=1.25) +got = a._calls["system_one_kwargs"] +check("hooks/forwarded to system_one", sorted(got), sorted(HOOK_KEYS)) +check("hooks/the hook list reaches it there", got.get("hooks"), [sentinel]) +check("hooks/hooks_timeout reaches it there", got.get("hooks_timeout"), 1.25) + +# 6. what the scan forwards is the window texts, in scan order, not the caller's state object +a = make_agent(canned) +a.predict_long(LONG, Q) +states = a._calls["batch_states"] +check("states/several windows", len(states) > 1, True) +check("states/decoded text, not the caller's dict", + all(isinstance(s, str) for s in states) and states[0] == "w0_71", True) +check("states/they overlap", a._calls["batch_states"][1], "w36_107") + +# 7. forwarding the controls must not move a decision +plain = make_agent(canned).predict_long(LONG, Q) +with_hooks = make_agent(canned).predict_long(LONG, Q, hooks=[sentinel], hooks_timeout=9) +check("hooks/forwarding is decision-neutral", with_hooks, plain) + +# 8. a start hook that answers the document, on the engine's own evidence +DOC = {"model": "laya-rl-agent", + "answers": {"dept": {"type": "choice", "choice": "b", "answer_confidence": 0.9}, + "flag": {"type": "noul", "noul": 0.2, "answer_confidence": 0.8}}, + "usage": {"input_tokens": 3, "output_tokens": 0}} + +# The instrument for sections 8-8e: a predict_long call is captured rather than let propagate, so +# a case that ends in the wrong shape -- an answer that should have come back raising, or a +# rejection that should not have happened at all -- fails a named check instead of aborting the run +# and hiding every check after it. +def _attempt(fn): + """Call `fn`, returning (value, exception).""" + try: + return fn(), None + except BaseException as exc: # noqa: BLE001 + return None, exc + + +def _kind(exc): + return exc.__class__.__name__ if exc else None + + +# The control first: how many windows this state really splits into, measured from the forward +# passes it costs rather than from the key under test. Two questions, one row each per window. +scan = make_real_agent() +base, base_exc = _attempt(lambda: scan.predict_long(LONG, Q)) +check("scan/an unhooked scan returns", _kind(base_exc), None) +nwin = sum(scan._forward_calls) // 2 +check("scan/more than one window", nwin > 1, True) +check("scan/the scan reports the windows it read", + ((base or {}).get("usage") or {}).get("windows", ""), nwin) +check("scan/the deciding window is the last", + ((base or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1) + + +def _narrow(keep): + """A start hook that truncates the window list and lets inference run on what is left.""" + def hook(ctx): + ctx.states = list(ctx.states)[:keep] + return hook + + +a = make_real_agent() +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + res, skip_exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.skip([DOC]))) +check("skip/one payload for the document is never refused as a count mismatch", _kind(skip_exc), None) +check("skip/no forward pass ran", a._forward_calls, []) +usage, answers = (res or {}).get("usage", {}), (res or {}).get("answers", {}) +check("skip/no window was scored", usage.get("windows", ""), 0) +check("skip/the payload tokens are kept", usage.get("input_tokens", ""), 3) +check("skip/the hook's answer is returned", answers.get("dept", {}).get("choice"), "b") +check("skip/no deciding window is claimed", + sorted(k for v in answers.values() for k in v if k == "window"), []) +check("skip/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"]) + +# 8b. a start hook that replaces the scan is the middle row: inference ran on the hook's states, so +# the answers aggregate, the count is theirs, and no span of the caller's document is named -- the +# offsets computed above describe windows that were not scored. +a = make_real_agent() +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + narrow, narrow_exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(1))) +check("narrow/one state left behind is a scan, not a rejected count", _kind(narrow_exc), None) +# Why the forward sits next to the count: it is the proof that tokens were spent, which is exactly +# what made this case read as a hook answer before. +check("narrow/inference ran on the state that was left", a._forward_calls, [2]) +check("narrow/the count is the state scored, not the split made", + ((narrow or {}).get("usage") or {}).get("windows", ""), 1) +check("narrow/the tokens that forward booked are kept", + ((narrow or {}).get("usage") or {}).get("input_tokens", 0) > 0, True) +check("narrow/no span of the caller's document is named", + sorted(k for v in (narrow or {}).get("answers", {}).values() for k in v if k == "window"), []) +check("narrow/the answer is still aggregated", + (narrow or {}).get("answers", {}).get("dept", {}).get("type"), "choice") +check("narrow/a rewrite is not warned about as a hook answer", + [w.category.__name__ for w in caught], []) +a = make_real_agent() +two, exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(2))) +check("narrow/two states left behind are counted as two", + ((two or {}).get("usage") or {}).get("windows", ""), 2) +check("narrow/and two states is what the forward saw", a._forward_calls, [4]) +# the other direction, and in place: `predict_long` hands the hook a copy of its split, so the +# state the hook invented is scored and counted without moving the windows computed above. +a = make_real_agent() +grown, exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.append("invented"))) +check("grow/scoring one invented state is not an error", _kind(exc), None) +check("grow/the hook's added state is scored", + ((grown or {}).get("usage") or {}).get("windows", ""), nwin + 1) +check("grow/the caller's split is the one that was extended", a._forward_calls, [2 * (nwin + 1)]) +check("grow/and no span is named", + sorted(k for v in (grown or {}).get("answers", {}).values() for k in v if k == "window"), []) + +# 8c. an unchanged scan keeps the attribution; any rewrite drops it, even one of the same length +a = make_real_agent() +noop, exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.extend(()))) +check("rewrite/a no-op on the list changes nothing", _kind(exc), None) +check("rewrite/a no-op keeps the count", ((noop or {}).get("usage") or {}).get("windows", ""), nwin) +check("rewrite/a no-op keeps the deciding window", + ((noop or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1) +a = make_real_agent() +upper, exc = _attempt(lambda: a.predict_long( + LONG, Q, on_predict_start=lambda ctx: setattr(ctx, "states", [s.upper() for s in ctx.states]))) +check("rewrite/the hook's text is what was scored", (a._encoded or [""])[0].isupper(), True) +check("rewrite/the same length is still the same count", + ((upper or {}).get("usage") or {}).get("windows", ""), nwin) +check("rewrite/a text rewrite names no span", + sorted(k for v in (upper or {}).get("answers", {}).values() for k in v if k == "window"), []) + +# 8e. a hook that leaves nothing scores nothing: 0 windows and no answers, not a max() over [] +a = make_real_agent() +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + empty, exc = _attempt( + lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: setattr(ctx, "states", []))) +check("empty/nothing scored returns instead of raising", _kind(exc), None) +check("empty/no answer is invented", (empty or {"answers": "?"}).get("answers", "?"), {}) +check("empty/the count says so", ((empty or {}).get("usage") or {}).get("windows", ""), 0) +check("empty/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"]) + +# 8d. the fits-in-one-window path answers to a hook too, so 0 keeps meaning "no window scored this" +a = make_real_agent() +short_hooked, exc = _attempt( + lambda: a.predict_long({"body": "x" * 50}, Q, on_predict_start=lambda ctx: ctx.skip([DOC]))) +check("short skip/no forward pass", a._forward_calls, []) +check("short skip/a hook answer is not counted as a window read", + ((short_hooked or {}).get("usage") or {}).get("windows", ""), 0) +a = make_real_agent() +short_plain, exc = _attempt(lambda: a.predict_long({"body": "x" * 50}, Q)) +check("short skip/the same state without a hook is one window", + ((short_plain or {}).get("usage") or {}).get("windows", ""), 1) + +# 9. a batch call that returns the wrong count for its own states is an error, not a guess +a = make_agent(lambda s, q: [{"answers": {}, "usage": {}}] * 2) +check_raises("count/rejects a result count matching nothing", ValueError, + lambda: a.predict_long(LONG, Q)) + +# 10. the count is total, so one question works on every result predict_long can return +short_all = make_agent(lambda s, q: []).predict_long({"body": "x" * 50}, Q) +scan_agent = make_agent(canned) +scanned_all = scan_agent.predict_long(LONG, Q) +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + hook_all, hook_exc = _attempt( + lambda: make_real_agent().predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.skip([DOC]))) +check("total/a hook answer returns rather than being refused", _kind(hook_exc), None) +check("total/no path leaves the key out", + [(r or {}).get("usage", {}).get("windows", "") + for r in (short_all, scanned_all, hook_all)], + [1, len(scan_agent._calls["batch_states"]), 0]) +check("total/0 separates a hook answer from a single-window one", + ((hook_all or {}).get("usage", {}).get("windows", ""), + short_all["usage"].get("windows", "")), + (0, 1)) + + +# 11. the page that teaches this key teaches the value the code actually writes +README = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "README.md"), encoding="utf-8").read() +bullet = README[README.index("A state that already fits one window"):][:600] +check("docs/README pins the single-window value", 'usage["windows"] = 1' in bullet, True) +check("docs/README states the key is total", "The key is total" in bullet, True) +check("docs/README documents all three counts", sorted(set(re.findall(r"`([0-9N])`", bullet))), + ["0", "1", "N"]) +# The two rules this branch added, pinned where they are taught rather than in the code comment +hooks_bullet = README[README.index("Hooks wrap the inference that answers the state"):][:900] +check("docs/README says attribution survives only an unchanged scan", + "reported when the scan reached inference unchanged" in hooks_bullet, True) +check("docs/README names ctx.skip as the way to answer", "ctx.skip(...)" in hooks_bullet, True) +API = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "docs", "hooks", "api.md"), encoding="utf-8").read() +api_para = API[API.index("On `predict_long` the hooks wrap"):][:1600] +check("docs/api.md tabulates the rewritten scan as its own case", + "| replaced the scan, in any way |" in api_para, True) +check("docs/api.md tabulates a hook answer as zero windows", + "| answered with `ctx.skip([result])` | `0` |" in api_para, True) +check("docs/predict_long's docstring says the key is total", + "always present" in (Agent.predict_long.__doc__ or ""), True) + print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) for f in FAIL: print(" FAIL " + f)