Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ jobs:
run: |
python tests/test_router.py
python tests/test_criteria.py
python tests/test_batch.py
python tests/test_download.py
python tests/test_shortlist.py
python tests/test_decision_model.py
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ jobs:
run: |
python tests/test_router.py
python tests/test_criteria.py
python tests/test_batch.py
python tests/test_email.py
python tests/test_download.py
python tests/test_shortlist.py
Expand Down
23 changes: 23 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,29 @@ Passing an empty question dictionary to `agent.predict(state, {})` or
and `"usage": {"input_tokens": 0, "output_tokens": 0}`. The state is not tokenized
and no model forward pass runs.

### Batch Mode: score many states in one forward pass

`predict` handles one state per call, which leaves most of the GPU's batch dimension idle. When you
have a list of items to score against the *same* questions — a backlog of tickets, a table of rows,
a log slice — `predict_batch` packs them into shared forward passes:

```python
states = [{"body": t} for t in ticket_texts] # a list of states

results = agent.predict_batch(states, questions) # one forward pass for the whole list
# results[i] is exactly what agent.predict(states[i], questions) would return

# Bound peak memory when the list (or the texts) are large — chunk into passes of N:
results = agent.predict_batch(states, questions, batch_size=64)
```

Results are aligned with `states` by index and identical in shape to `predict`. Decisions match the
one-at-a-time path exactly (numbers are bit-identical on CPU; on GPU they can differ in the 4th
decimal because fp16 autocast reorders reductions across padding widths). Batching is a **GPU
throughput win** — on an RTX 5060 Ti, per-decision latency drops from ~10 ms one-by-one to ~1 ms
batched (measured ~9–10×). On CPU the model is already compute-bound, so batching does not speed it
up; use it there only for API convenience.

---

## GPU Fast Path (TileLang)
Expand Down
171 changes: 113 additions & 58 deletions laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import os
import threading
import warnings
from typing import Any, Dict, Optional, Union
from typing import Any, Dict, List, Optional, Union

import numpy as np
import torch
Expand Down Expand Up @@ -416,89 +416,59 @@ def _to_internal(qdef: Dict) -> Dict:
q["labels"] = qdef["labels"]
return q

@torch.no_grad()
def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
"""Evaluate typed questions across state in a single, parallel forward pass.

Args:
state: Text string, JSON dict, or conversation turn list.
questions: Dictionary mapping question_id -> question definition.
- choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}
- score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}
- noul: {"type": "noul", "instructions": "...",
"criteria": {"false": "...", "true": "..."},
"labels": {"false": "B", "true": "A"}}

Noul criteria and labels are optional. Labels only control the text shown to the
model; their keys retain false/true semantics, and the returned `noul` value is
always P(true). Labels default to false/true for compatibility.

Returns:
Dictionary with answers, probabilities, calibrated confidence, and token usage.
Empty questions return empty answers and zero token usage without tokenization
or a model forward pass.
"""
ids = list(questions.keys())
if not ids:
return {
"model": "laya-rl-agent",
"answers": {},
"usage": {"input_tokens": 0, "output_tokens": 0},
}
items = []
def _encode_state(self, state: Union[str, dict, list], ids: List[str], internal: Dict[str, Dict]) -> List[Dict]:
"""Tokenize one state against every (already validated + normalized) question."""
max_len = self.cfg.get("max_len", 512)
head_max_len = self.cfg.get("head_max_len", 192)
# A chronological conversation list is serialized newest-last, so the default
# right-truncation (st[:room]) would silently drop the newest turn. Truncate
# from the left for lists so the most recent intent is preserved.
truncate_left = isinstance(state, list)

items = []
for qid in ids:
self._check_question(qid, questions[qid])
q = self._to_internal(questions[qid])
q = internal[qid]
seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len,
truncate_left=truncate_left)
if len(markers) != len(render_options(q)):
raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})
return items

b = collate_items([items], self.tok.pad_token_id)
use_amp = self.device.type == "cuda"

try:
def _forward(self, b: Dict):
"""Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
def run():
use_amp = self.device.type == "cuda"
with torch.autocast(device_type=self.device.type, dtype=self.dtype, enabled=use_amp):
logits, act = self.model(
return self.model(
b["input_ids"].to(self.device),
b["attention_mask"].to(self.device),
b["marker_pos"].to(self.device),
b["marker_mask"].to(self.device),
b["qtype"].to(self.device),
)

try:
logits, act = run()
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
if self.device.type != "cpu" and ("memory" in str(e).lower() or "cuda" in str(e).lower()):
print("Warning: GPU memory exceeded during inference. Falling back to CPU...")
self.device = torch.device("cpu")
self.dtype = torch.float32
self.model.to(self.device)
logits, act = self.model(
b["input_ids"].to(self.device),
b["attention_mask"].to(self.device),
b["marker_pos"].to(self.device),
b["marker_mask"].to(self.device),
b["qtype"].to(self.device),
)
logits, act = run()
else:
raise e

logits = logits.float().cpu().numpy()
act = torch.softmax(act.float(), -1).cpu().numpy()
return logits.float().cpu().numpy(), torch.softmax(act.float(), -1).cpu().numpy()

def _decode_answers(self, logits, act, items: List[Dict], ids: List[str],
internal: Dict[str, Dict], offset: int) -> Dict[str, Any]:
"""Turn one state's logit rows (starting at `offset`) into typed answers."""
answers = {}
n_tokens = int(b["attention_mask"].sum())

for r, qid in enumerate(ids):
q = self._to_internal(questions[qid])
k = len(items[r]["markers"])
for j, qid in enumerate(ids):
r = offset + j
q = internal[qid]
k = len(items[j]["markers"])
qt = QTYPES[q["t"]]
t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])
z = logits[r, :k] / t_scale
Expand Down Expand Up @@ -534,12 +504,97 @@ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[st
"confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),
"action": ext,
}
return answers

@torch.no_grad()
def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],
batch_size: Optional[int] = None) -> List[Dict[str, Any]]:
"""Evaluate the same questions over many states, packing them into shared forward passes.

This is the throughput path. `system_one`/`predict` handle one state per forward pass; on a
GPU that leaves most of the batch dimension idle. `predict_batch` collates several states'
question rows into one tensor, so a call that would take N sequential forward passes takes
one (or `ceil(len(states) / batch_size)`), which is several times faster per decision on GPU.

Args:
states: A list of states (each a text string, JSON dict, or conversation turn list).
The same `questions` are evaluated against every state.
questions: Question definitions, exactly as accepted by `system_one`.
batch_size: Optional cap on states per forward pass. `None` sends them all in one pass;
set it to bound peak memory when batching many or long states.

Returns:
A list of per-state result dicts, each identical in shape to `system_one`'s output and
aligned with `states` by index.
"""
if isinstance(states, (str, bytes, dict)):
raise TypeError(
"predict_batch expects a list of states; pass a single state to predict()/system_one()."
)
states = list(states)
if not states:
return []

ids = list(questions.keys())
# Empty questions: empty answers, zero usage, no tokenization or forward (matches system_one).
if not ids:
return [{"model": "laya-rl-agent", "answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}}
for _ in states]

# Validate + normalize each question once (state-independent), preserving system_one's checks.
for qid in ids:
self._check_question(qid, questions[qid])
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
chunk = batch_size if (batch_size and batch_size > 0) else len(states)

results: List[Dict[str, Any]] = []
for start in range(0, len(states), chunk):
part = states[start:start + chunk]
per_state_items = [self._encode_state(st, ids, internal) for st in part]

b = collate_items(per_state_items, self.tok.pad_token_id)
logits, act = self._forward(b)
att = b["attention_mask"]

row = 0
for items in per_state_items:
nrows = len(items)
n_tokens = int(att[row:row + nrows].sum())
answers = self._decode_answers(logits, act, items, ids, internal, row)
results.append({
"model": "laya-rl-agent",
"answers": answers,
"usage": {"input_tokens": n_tokens, "output_tokens": 0},
})
row += nrows

return results

@torch.no_grad()
def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
"""Evaluate typed questions across state in a single, parallel forward pass.

return {
"model": "laya-rl-agent",
"answers": answers,
"usage": {"input_tokens": n_tokens, "output_tokens": 0},
}
Args:
state: Text string, JSON dict, or conversation turn list.
questions: Dictionary mapping question_id -> question definition.
- choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}
- score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}
- noul: {"type": "noul", "instructions": "...",
"criteria": {"false": "...", "true": "..."},
"labels": {"false": "B", "true": "A"}}

Noul criteria and labels are optional. Labels only control the text shown to the
model; their keys retain false/true semantics, and the returned `noul` value is
always P(true). Labels default to false/true for compatibility.

Returns:
Dictionary with answers, probabilities, calibrated confidence, and token usage.
Empty questions return empty answers and zero token usage without tokenization
or a model forward pass.

To score many states at once, see `predict_batch`, which shares forward passes across them.
"""
return self.predict_batch([state], questions)[0]

def __enter__(self):
return self
Expand Down
125 changes: 125 additions & 0 deletions tests/test_batch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
"""Batch inference: predict_batch packs many states into shared forward passes.

These tests need no model weights. They cover two things:
1. Contract — predict_batch exists and system_one is defined in terms of it, so the single-state
and batched paths can never numerically drift apart.
2. Orchestration — empty input, bad input, per-state row mapping, and batch_size chunking, all
exercised against a fake whose forward is stubbed. The numerical equivalence of the real
model path (predict_batch(states) == [system_one(s) for s in states]) is checked with weights
in tests/test_local_e2e.py.
"""
import inspect
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import numpy as np # noqa: E402

from laya import agent as _agent # noqa: E402
from laya.agent import Agent # noqa: E402

PASS, FAIL = [], []


def check(name, got, want):
if got == want:
PASS.append(name)
else:
FAIL.append("%s:\n got %r\n want %r" % (name, got, want))


def check_true(name, cond, detail=""):
if cond:
PASS.append(name)
else:
FAIL.append("%s %s" % (name, detail))


def check_raises(name, exc, fn):
try:
fn()
except exc:
PASS.append(name)
except Exception as e: # noqa: BLE001
FAIL.append("%s: raised %r, expected %s" % (name, e, exc.__name__))
else:
FAIL.append("%s: did not raise %s" % (name, exc.__name__))


# --------------------------------------------------------------- contract (source inspection)
check_true("contract/predict_batch is defined", callable(getattr(Agent, "predict_batch", None)))

_so = inspect.getsource(Agent.system_one)
check_true("contract/system_one delegates to predict_batch", "self.predict_batch(" in _so)
check_true("contract/predict alias preserved", Agent.predict is Agent.system_one)

_sig = inspect.signature(Agent.predict_batch)
check_true("contract/predict_batch has batch_size kwarg", "batch_size" in _sig.parameters)


# --------------------------------------------------------------- orchestration (weight-free fake)
# We drive the real predict_batch method but stub the three helpers it composes, so no encoder,
# tokenizer vocabulary, or weights are needed. collate_items runs for real on the fake items.
NQ = 2
QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}


def make_fake():
fake = _agent.Agent.__new__(_agent.Agent)
fake.tok = type("Tok", (), {"pad_token_id": 0})()
fake._to_internal = staticmethod(Agent._to_internal).__func__ # reuse the real normalizer
fake._forward_calls = []

def _encode_state(state, ids, internal):
# one 3-token, 2-marker item per question; content is irrelevant to the mapping test
return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]

def _forward(b):
n = b["input_ids"].shape[0]
fake._forward_calls.append(n)
# deterministic logits so decode is stable; act pre-softmaxed as _forward would return it
return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32)

def _decode_answers(logits, act, items, ids, internal, offset):
# record the row offset this state was decoded from, to verify per-state alignment
return {"_offset": offset}

fake._encode_state = _encode_state
fake._forward = _forward
fake._decode_answers = _decode_answers
return fake


f = make_fake()
check("orchestration/empty states -> empty list", f.predict_batch([], QUESTIONS), [])

f = make_fake()
res = f.predict_batch(["s0", "s1", "s2"], QUESTIONS)
check("orchestration/one result per state", len(res), 3)
check("orchestration/state 0 decoded from row 0", res[0]["answers"]["_offset"], 0)
check("orchestration/state 1 decoded from row NQ", res[1]["answers"]["_offset"], NQ)
check("orchestration/state 2 decoded from row 2*NQ", res[2]["answers"]["_offset"], 2 * NQ)
check_true("orchestration/result shape matches system_one",
all(set(r) == {"model", "answers", "usage"} for r in res))
check("orchestration/single forward pass by default", f._forward_calls, [6]) # 3 states x 2 questions

f = make_fake()
f.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=1)
check("orchestration/batch_size=1 chunks into three passes", f._forward_calls, [2, 2, 2])

f = make_fake()
f.predict_batch(["s0", "s1", "s2", "s3", "s4"], QUESTIONS, batch_size=2)
check("orchestration/batch_size=2 chunks 5 states as 2+2+1", f._forward_calls, [4, 4, 2])

f = make_fake()
check_raises("orchestration/bare string rejected", TypeError, lambda: f.predict_batch("just a string", QUESTIONS))
f = make_fake()
check_raises("orchestration/bare dict rejected", TypeError, lambda: f.predict_batch({"body": "x"}, QUESTIONS))


# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL:
print(" FAIL " + f_)
sys.exit(1 if FAIL else 0)
Loading