From bc1546481f4ad2531e4d5cc9b0c632764ccab734 Mon Sep 17 00:00:00 2001 From: AlKor13 Date: Mon, 21 Sep 2026 12:24:37 +0500 Subject: [PATCH] feat(agent): add predict_batch for shared-forward-pass batch inference `predict`/`system_one` evaluate one state per forward pass, which leaves most of the GPU batch dimension idle. `predict_batch(states, questions)` collates many states' question rows into shared forward passes, so a call that would take N sequential passes takes one (or `ceil(N / batch_size)`). Measured on an RTX 5060 Ti (mmBERT-base, 30 states x 3 questions): per-decision latency drops from ~10 ms one-by-one to ~1 ms batched (~9-10x). CPU is already compute-bound, so batching there is for API convenience, not speed. To guarantee the batched and single-state paths cannot drift, `system_one` is now `predict_batch([state], questions)[0]` -- one code path. The shared forward and answer-decoding logic are factored into `_encode_state`, `_forward` (keeping the GPU->CPU OOM fallback), and `_decode_answers`. Results are byte-identical to the old single path on CPU (fp32); on GPU they match to <5e-3 (fp16 autocast reorders reductions across padding widths) with identical decisions and token counts. - tests/test_batch.py: weight-free contract + orchestration tests (empty input, bad input, per-state row mapping, batch_size chunking); wired into CI. - tests/test_local_e2e.py: real-weights section verifying predict_batch matches system_one decision-for-decision across full/chunked/one-by-one paths. - README: Batch Mode section. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 1 + .github/workflows/release.yml | 1 + README.md | 23 +++++ laya/agent.py | 159 +++++++++++++++++++++++----------- tests/test_batch.py | 125 ++++++++++++++++++++++++++ tests/test_local_e2e.py | 75 ++++++++++++++++ 6 files changed, 332 insertions(+), 52 deletions(-) create mode 100644 tests/test_batch.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a06e70ec..05eccb4e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: run: | python tests/test_router.py python tests/test_criteria.py + python tests/test_batch.py python tests/test_download.py python tests/test_shortlist.py python tests/test_decision_model.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40ce7392b..d04f6f2d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,7 @@ jobs: run: | python tests/test_router.py python tests/test_criteria.py + python tests/test_batch.py python tests/test_email.py python tests/test_download.py python tests/test_shortlist.py diff --git a/README.md b/README.md index a11bdaec8..42631973d 100644 --- a/README.md +++ b/README.md @@ -387,6 +387,29 @@ Passing an empty question dictionary to `agent.predict(state, {})` or and `"usage": {"input_tokens": 0, "output_tokens": 0}`. The state is not tokenized and no model forward pass runs. +### Batch Mode: score many states in one forward pass + +`predict` handles one state per call, which leaves most of the GPU's batch dimension idle. When you +have a list of items to score against the *same* questions — a backlog of tickets, a table of rows, +a log slice — `predict_batch` packs them into shared forward passes: + +```python +states = [{"body": t} for t in ticket_texts] # a list of states + +results = agent.predict_batch(states, questions) # one forward pass for the whole list +# results[i] is exactly what agent.predict(states[i], questions) would return + +# Bound peak memory when the list (or the texts) are large — chunk into passes of N: +results = agent.predict_batch(states, questions, batch_size=64) +``` + +Results are aligned with `states` by index and identical in shape to `predict`. Decisions match the +one-at-a-time path exactly (numbers are bit-identical on CPU; on GPU they can differ in the 4th +decimal because fp16 autocast reorders reductions across padding widths). Batching is a **GPU +throughput win** — on an RTX 5060 Ti, per-decision latency drops from ~10 ms one-by-one to ~1 ms +batched (measured ~9–10×). On CPU the model is already compute-bound, so batching does not speed it +up; use it there only for API convenience. + --- ## Automated Confidence Gating diff --git a/laya/agent.py b/laya/agent.py index 70163bf98..0ea5bbdbc 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union import numpy as np import torch @@ -315,78 +315,54 @@ def _to_internal(qdef: Dict) -> Dict: ins = json.dumps(ins) return {"t": t, "ins": ins, "crit": crit} - @torch.no_grad() - def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: - """Evaluate typed questions across state in a single, parallel forward pass. - - Args: - state: Text string, JSON dict, or conversation turn list. - questions: Dictionary mapping question_id -> question definition. - - choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}} - - score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]} - - noul: {"type": "noul", "instructions": "...", "criteria": {"true": "...", "false": "..."}} - - Returns: - Dictionary with answers, probabilities, calibrated confidence, and token usage. - Empty questions return empty answers and zero token usage without tokenization - or a model forward pass. - """ - ids = list(questions.keys()) - if not ids: - return { - "model": "laya-rl-agent", - "answers": {}, - "usage": {"input_tokens": 0, "output_tokens": 0}, - } - items = [] + def _encode_state(self, state: Union[str, dict, list], ids: List[str], internal: Dict[str, Dict]) -> List[Dict]: + """Tokenize one state against every (already validated + normalized) question.""" max_len = self.cfg.get("max_len", 512) head_max_len = self.cfg.get("head_max_len", 192) - + items = [] for qid in ids: - self._check_question(qid, questions[qid]) - q = self._to_internal(questions[qid]) + q = internal[qid] seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len) if len(markers) != len(render_options(q)): raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len)) items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]}) + return items - b = collate_items([items], self.tok.pad_token_id) - use_amp = self.device.type == "cuda" - - try: + def _forward(self, b: Dict): + """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs.""" + def run(): + use_amp = self.device.type == "cuda" with torch.autocast(device_type=self.device.type, dtype=self.dtype, enabled=use_amp): - logits, act = self.model( + return self.model( b["input_ids"].to(self.device), b["attention_mask"].to(self.device), b["marker_pos"].to(self.device), b["marker_mask"].to(self.device), b["qtype"].to(self.device), ) + + try: + logits, act = run() except (RuntimeError, torch.cuda.OutOfMemoryError) as e: if self.device.type != "cpu" and ("memory" in str(e).lower() or "cuda" in str(e).lower()): print("Warning: GPU memory exceeded during inference. Falling back to CPU...") self.device = torch.device("cpu") self.dtype = torch.float32 self.model.to(self.device) - logits, act = self.model( - b["input_ids"].to(self.device), - b["attention_mask"].to(self.device), - b["marker_pos"].to(self.device), - b["marker_mask"].to(self.device), - b["qtype"].to(self.device), - ) + logits, act = run() else: raise e - logits = logits.float().cpu().numpy() - act = torch.softmax(act.float(), -1).cpu().numpy() + return logits.float().cpu().numpy(), torch.softmax(act.float(), -1).cpu().numpy() + def _decode_answers(self, logits, act, items: List[Dict], ids: List[str], + internal: Dict[str, Dict], offset: int) -> Dict[str, Any]: + """Turn one state's logit rows (starting at `offset`) into typed answers.""" answers = {} - n_tokens = int(b["attention_mask"].sum()) - - for r, qid in enumerate(ids): - q = self._to_internal(questions[qid]) - k = len(items[r]["markers"]) + for j, qid in enumerate(ids): + r = offset + j + q = internal[qid] + k = len(items[j]["markers"]) qt = QTYPES[q["t"]] t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt]) z = logits[r, :k] / t_scale @@ -422,12 +398,91 @@ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[st "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4), "action": ext, } + return answers + + @torch.no_grad() + def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]], + batch_size: Optional[int] = None) -> List[Dict[str, Any]]: + """Evaluate the same questions over many states, packing them into shared forward passes. - return { - "model": "laya-rl-agent", - "answers": answers, - "usage": {"input_tokens": n_tokens, "output_tokens": 0}, - } + This is the throughput path. `system_one`/`predict` handle one state per forward pass; on a + GPU that leaves most of the batch dimension idle. `predict_batch` collates several states' + question rows into one tensor, so a call that would take N sequential forward passes takes + one (or `ceil(len(states) / batch_size)`), which is several times faster per decision on GPU. + + Args: + states: A list of states (each a text string, JSON dict, or conversation turn list). + The same `questions` are evaluated against every state. + questions: Question definitions, exactly as accepted by `system_one`. + batch_size: Optional cap on states per forward pass. `None` sends them all in one pass; + set it to bound peak memory when batching many or long states. + + Returns: + A list of per-state result dicts, each identical in shape to `system_one`'s output and + aligned with `states` by index. + """ + if isinstance(states, (str, bytes, dict)): + raise TypeError( + "predict_batch expects a list of states; pass a single state to predict()/system_one()." + ) + states = list(states) + if not states: + return [] + + ids = list(questions.keys()) + # Empty questions: empty answers, zero usage, no tokenization or forward (matches system_one). + if not ids: + return [{"model": "laya-rl-agent", "answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}} + for _ in states] + + # Validate + normalize each question once (state-independent), preserving system_one's checks. + for qid in ids: + self._check_question(qid, questions[qid]) + internal = {qid: self._to_internal(questions[qid]) for qid in ids} + chunk = batch_size if (batch_size and batch_size > 0) else len(states) + + results: List[Dict[str, Any]] = [] + for start in range(0, len(states), chunk): + part = states[start:start + chunk] + per_state_items = [self._encode_state(st, ids, internal) for st in part] + + b = collate_items(per_state_items, self.tok.pad_token_id) + logits, act = self._forward(b) + att = b["attention_mask"] + + row = 0 + for items in per_state_items: + nrows = len(items) + n_tokens = int(att[row:row + nrows].sum()) + answers = self._decode_answers(logits, act, items, ids, internal, row) + results.append({ + "model": "laya-rl-agent", + "answers": answers, + "usage": {"input_tokens": n_tokens, "output_tokens": 0}, + }) + row += nrows + + return results + + @torch.no_grad() + def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + """Evaluate typed questions across state in a single, parallel forward pass. + + Args: + state: Text string, JSON dict, or conversation turn list. + questions: Dictionary mapping question_id -> question definition. + - choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}} + - score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]} + - noul: {"type": "noul", "instructions": "...", "criteria": {"true": "...", "false": "..."}} + + Returns: + Dictionary with answers, probabilities, calibrated confidence, and token usage. + Empty questions return empty answers and zero token usage without tokenization + or a model forward pass. + + To score many states at once, see `predict_batch`, which shares forward passes across them. + """ + return self.predict_batch([state], questions)[0] def __enter__(self): return self diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 000000000..ca9041e58 --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,125 @@ +"""Batch inference: predict_batch packs many states into shared forward passes. + +These tests need no model weights. They cover two things: + 1. Contract — predict_batch exists and system_one is defined in terms of it, so the single-state + and batched paths can never numerically drift apart. + 2. Orchestration — empty input, bad input, per-state row mapping, and batch_size chunking, all + exercised against a fake whose forward is stubbed. The numerical equivalence of the real + model path (predict_batch(states) == [system_one(s) for s in states]) is checked with weights + in tests/test_local_e2e.py. +""" +import inspect +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import numpy as np # noqa: E402 + +from laya import agent as _agent # noqa: E402 +from laya.agent import Agent # noqa: E402 + +PASS, FAIL = [], [] + + +def check(name, got, want): + if got == want: + PASS.append(name) + else: + FAIL.append("%s:\n got %r\n want %r" % (name, got, want)) + + +def check_true(name, cond, detail=""): + if cond: + PASS.append(name) + else: + FAIL.append("%s %s" % (name, detail)) + + +def check_raises(name, exc, fn): + try: + fn() + except exc: + PASS.append(name) + except Exception as e: # noqa: BLE001 + FAIL.append("%s: raised %r, expected %s" % (name, e, exc.__name__)) + else: + FAIL.append("%s: did not raise %s" % (name, exc.__name__)) + + +# --------------------------------------------------------------- contract (source inspection) +check_true("contract/predict_batch is defined", callable(getattr(Agent, "predict_batch", None))) + +_so = inspect.getsource(Agent.system_one) +check_true("contract/system_one delegates to predict_batch", "self.predict_batch(" in _so) +check_true("contract/predict alias preserved", Agent.predict is Agent.system_one) + +_sig = inspect.signature(Agent.predict_batch) +check_true("contract/predict_batch has batch_size kwarg", "batch_size" in _sig.parameters) + + +# --------------------------------------------------------------- orchestration (weight-free fake) +# We drive the real predict_batch method but stub the three helpers it composes, so no encoder, +# tokenizer vocabulary, or weights are needed. collate_items runs for real on the fake items. +NQ = 2 +QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}} + + +def make_fake(): + fake = _agent.Agent.__new__(_agent.Agent) + fake.tok = type("Tok", (), {"pad_token_id": 0})() + fake._to_internal = staticmethod(Agent._to_internal).__func__ # reuse the real normalizer + fake._forward_calls = [] + + def _encode_state(state, ids, internal): + # one 3-token, 2-marker item per question; content is irrelevant to the mapping test + return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids] + + def _forward(b): + n = b["input_ids"].shape[0] + fake._forward_calls.append(n) + # deterministic logits so decode is stable; act pre-softmaxed as _forward would return it + return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32) + + def _decode_answers(logits, act, items, ids, internal, offset): + # record the row offset this state was decoded from, to verify per-state alignment + return {"_offset": offset} + + fake._encode_state = _encode_state + fake._forward = _forward + fake._decode_answers = _decode_answers + return fake + + +f = make_fake() +check("orchestration/empty states -> empty list", f.predict_batch([], QUESTIONS), []) + +f = make_fake() +res = f.predict_batch(["s0", "s1", "s2"], QUESTIONS) +check("orchestration/one result per state", len(res), 3) +check("orchestration/state 0 decoded from row 0", res[0]["answers"]["_offset"], 0) +check("orchestration/state 1 decoded from row NQ", res[1]["answers"]["_offset"], NQ) +check("orchestration/state 2 decoded from row 2*NQ", res[2]["answers"]["_offset"], 2 * NQ) +check_true("orchestration/result shape matches system_one", + all(set(r) == {"model", "answers", "usage"} for r in res)) +check("orchestration/single forward pass by default", f._forward_calls, [6]) # 3 states x 2 questions + +f = make_fake() +f.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=1) +check("orchestration/batch_size=1 chunks into three passes", f._forward_calls, [2, 2, 2]) + +f = make_fake() +f.predict_batch(["s0", "s1", "s2", "s3", "s4"], QUESTIONS, batch_size=2) +check("orchestration/batch_size=2 chunks 5 states as 2+2+1", f._forward_calls, [4, 4, 2]) + +f = make_fake() +check_raises("orchestration/bare string rejected", TypeError, lambda: f.predict_batch("just a string", QUESTIONS)) +f = make_fake() +check_raises("orchestration/bare dict rejected", TypeError, lambda: f.predict_batch({"body": "x"}, QUESTIONS)) + + +# --------------------------------------------------------------------------- report +print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) +for f_ in FAIL: + print(" FAIL " + f_) +sys.exit(1 if FAIL else 0) diff --git a/tests/test_local_e2e.py b/tests/test_local_e2e.py index 0bfd55050..70ce99d21 100644 --- a/tests/test_local_e2e.py +++ b/tests/test_local_e2e.py @@ -210,6 +210,81 @@ def head(t): ok("explicit typed-decisions honoured", res_td["routing"]["model"] == "typed-decisions") ok("routing payload serialises", isinstance(json.dumps(res_td["routing"]), str)) + +# ------------------------------------------------- 6. Batch inference matches one-by-one +head("6. predict_batch matches system_one, decision-for-decision (real forward passes)") +BATCH_STATES = [ + {"message": "I was charged twice for invoice 4411, please refund it today."}, + {"message": "The dashboard has been down for an hour and my team is blocked."}, + {"message": "What is the price of the enterprise plan? No rush at all."}, + {"message": "Cannot reset my password, the email never arrives."}, + {"message": "Thanks, everything is working great now!"}, +] +# The English checkpoint was freed above; load a fresh agent for this section. +ba = laya.load(LOCAL["multilingual"], device=DEVICE) +# fp16 autocast on GPU reorders reductions across padding widths, so numbers can wobble in the +# 4th decimal; CPU fp32 is exact. Decisions (argmax) must be identical either way. +ATOL = 5e-3 if str(ba.device) != "cpu" else 0.0 + +single = [ba.predict(s, QD) for s in BATCH_STATES] +batched = ba.predict_batch(BATCH_STATES, QD) +chunked = ba.predict_batch(BATCH_STATES, QD, batch_size=2) + +ok("batch returns one result per state", len(batched) == len(BATCH_STATES), + "got %d" % len(batched)) +ok("empty batch returns []", ba.predict_batch([], QD) == []) +try: + ba.predict_batch("a bare string", QD) + ok("bare state rejected", False, "no TypeError raised") +except TypeError: + ok("bare state rejected", True) + + +def _num_close(x, y, atol): + return abs(x - y) <= atol + + +def _answers_agree(a, b, atol): + if set(a) != set(b): + return False, "question ids differ" + for qid in a: + x, y = a[qid], b[qid] + if x["type"] != y["type"]: + return False, "%s: type" % qid + if x["type"] == "choice": + if x["choice"] != y["choice"]: + return False, "%s: choice %s vs %s" % (qid, x["choice"], y["choice"]) + for kk in x["probabilities"]: + if not _num_close(x["probabilities"][kk], y["probabilities"][kk], atol): + return False, "%s: prob[%s]" % (qid, kk) + elif x["type"] == "score": + if not _num_close(x["score"], y["score"], atol): + return False, "%s: score %s vs %s" % (qid, x["score"], y["score"]) + elif x["type"] == "noul": + if not _num_close(x["noul"], y["noul"], atol): + return False, "%s: noul %s vs %s" % (qid, x["noul"], y["noul"]) + return True, "" + + +bad = 0 +for i, (s, b) in enumerate(zip(single, batched)): + agree, why = _answers_agree(s["answers"], b["answers"], ATOL) + if not agree: + bad += 1 + print(" FAIL state %d: %s" % (i, why), flush=True) + if s["usage"]["input_tokens"] != b["usage"]["input_tokens"]: + bad += 1 + print(" FAIL state %d: token count %d vs %d" + % (i, s["usage"]["input_tokens"], b["usage"]["input_tokens"]), flush=True) +ok("batched == one-by-one (decisions + numbers within atol %.0e)" % ATOL, bad == 0, + "%d states diverged" % bad) + +bad_chunk = sum(0 if _answers_agree(b["answers"], c["answers"], ATOL)[0] else 1 + for b, c in zip(batched, chunked)) +ok("batch_size chunking matches one big pass", bad_chunk == 0, "%d diverged" % bad_chunk) +print(" verified %d states across full / chunked / one-by-one paths" % len(BATCH_STATES), flush=True) +del ba + # ---------------------------------------------------------------- summary head("SUMMARY") for n in NOTES: