Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,41 @@ def __init__(
" See https://pytorch.org/get-started/locally/\n"
% (fell_back_from, fell_back_why), flush=True)

@staticmethod
def _check_question(qid: str, qdef: Any) -> None:
"""Reject a question that cannot be answered, naming it and what to fix.

`render_options` reads `criteria` in the shape the question's type expects and the decision
head needs at least one option, so a malformed definition used to surface from three frames
down as something that names neither the question nor the problem: `AttributeError:
'NoneType' object has no attribute 'items'`, `KeyError: 'bool'`, or a `selected index k out
of range` raised inside the model for a question that ended up with no options at all.
"""
if not isinstance(qdef, dict):
raise ValueError("question %r: definition must be a dict, got %s"
% (qid, type(qdef).__name__))
t = qdef.get("type")
if t not in QTYPES:
raise ValueError("question %r: unknown type %r; use one of %s" % (qid, t, sorted(QTYPES)))
if "instructions" not in qdef:
raise ValueError("question %r: no 'instructions'; add the text the model should answer" % (qid,))
crit = qdef.get("criteria")
if t == "choice":
if not isinstance(crit, (dict, list)):
raise ValueError("question %r: a choice question takes 'criteria' as a dict of "
"label -> description, or a list of labels" % (qid,))
if not crit:
raise ValueError("question %r: a choice question needs at least one criterion" % (qid,))
elif t == "score":
if not isinstance(crit, list):
raise ValueError("question %r: a score question takes 'criteria' as a list of level "
"descriptions, index 0 first" % (qid,))
if not crit:
raise ValueError("question %r: a score question needs at least one level" % (qid,))
elif crit is not None and not isinstance(crit, dict):
raise ValueError("question %r: a noul question takes 'criteria' as a dict with optional "
"'true'/'false' descriptions, or omits it" % (qid,))

@staticmethod
def _to_internal(qdef: Dict) -> Dict:
t = qdef["type"]
Expand Down Expand Up @@ -282,6 +317,7 @@ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[st
head_max_len = self.cfg.get("head_max_len", 192)

for qid in ids:
self._check_question(qid, questions[qid])
q = self._to_internal(questions[qid])
seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len)
if len(markers) != len(render_options(q)):
Expand Down
129 changes: 129 additions & 0 deletions tests/test_criteria.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@
Regression tests for the bug reported in PR #2 (trocker): a `noul` question whose criteria
values were dicts raised `TypeError: can only concatenate str (not "dict") to str`, and
`choice`/`score` stringified dicts as Python reprs instead of JSON.

A question whose *shape* is wrong is the other half of the same promise and lives at the bottom of
this file: `Agent.system_one` rejects it by name before anything is rendered or tokenized, instead
of raising `AttributeError: 'NoneType' object has no attribute 'items'` from `render_options` or a
`selected index k out of range` from inside the decision head (#182).
"""
import json
import os
Expand Down Expand Up @@ -116,6 +121,130 @@ def check_true(name, cond, detail=""):
"torch.cuda.is_available() or getattr(torch.version" not in _src)


# --------------------------------------------------------------- malformed question shapes (#182)
# A question that cannot be answered used to fail three frames down, as an exception that named
# neither the question nor the fix: `AttributeError: 'NoneType' object has no attribute 'items'`
# from `render_options` for a `choice` without criteria, `KeyError: 'bool'` from the type table,
# and -- for a question that ended up with no options at all -- a `selected index k out of range`
# raised inside `DecisionModel.forward`, which reads like a bug in laya rather than in the caller's
# definition. The inference path below is real: a tiny from-config encoder, no checkpoint
# downloaded (tests/test_local_e2e.py covers the real weights).
import torch # noqa: E402
from transformers import AutoConfig, AutoModel # noqa: E402

from laya.agent import Agent # noqa: E402
from laya.common import DecisionModel # noqa: E402
from laya.router import Router # noqa: E402


class _FakeTok:
"""The tokenizer surface `build_sequence` uses, with predictable ids."""
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2
mask_token = "[MASK]"

def __call__(self, text, add_special_tokens=False):
return {"input_ids": [10 + (len(w) % 90) for w in text.split() if w]}


def _tiny_agent():
"""An `Agent` with a tiny random encoder: the inference path, without the download."""
agent = object.__new__(Agent)
cfg = AutoConfig.for_model("bert", hidden_size=16, num_hidden_layers=1, num_attention_heads=1,
intermediate_size=32, vocab_size=64)
agent.cfg = {"max_len": 256, "head_max_len": 96, "encoder": "tiny"}
agent.tok = _FakeTok()
agent.model = DecisionModel(AutoModel.from_config(cfg), head_layers=1, n_act=2).eval()
agent.device = torch.device("cpu")
agent.dtype = torch.float32
agent.temperature = [1.0, 1.0, 1.0]
agent.temperature_by_options = {}
return agent


STATE = {"body": "I was charged twice for invoice 4411 and want a refund"}
agent = _tiny_agent()
for label, qdef in [
("choice without criteria", {"type": "choice", "instructions": "Which team?"}),
("choice with criteria None", {"type": "choice", "instructions": "Which team?", "criteria": None}),
("choice with empty criteria", {"type": "choice", "instructions": "Which team?", "criteria": {}}),
("choice with a tuple of labels", {"type": "choice", "instructions": "Which team?",
"criteria": ("billing", "tech")}),
("score without criteria", {"type": "score", "instructions": "How urgent?"}),
("score with an empty list", {"type": "score", "instructions": "How urgent?", "criteria": []}),
("score with a dict of levels", {"type": "score", "instructions": "How urgent?",
"criteria": {"low": "no pressure", "high": "blocking"}}),
("noul with list criteria", {"type": "noul", "instructions": "Is it spam?", "criteria": ["a", "b"]}),
("noul with string criteria", {"type": "noul", "instructions": "Is it spam?", "criteria": "spam?"}),
("unknown type", {"type": "bool", "instructions": "Is it spam?"}),
("missing type", {"instructions": "Is it spam?"}),
("no instructions", {"type": "noul"}),
]:
try:
agent.system_one(STATE, {"q": qdef})
FAIL.append("rejected/%s: no error raised" % label)
except ValueError as e:
# the message must name the question: a caller with twenty of them needs to know which
check_true("rejected/%s names the question" % label, "'q'" in str(e), str(e))
check_true("rejected/%s says what to fix" % label, len(str(e)) > 40, str(e))
except Exception as e:
FAIL.append("rejected/%s: %s instead of ValueError: %s" % (label, type(e).__name__, e))

# the same questions through the public entry point, not only the method under it
router = Router()
router.attach("english", agent)
for label, qdef in (("choice without criteria", {"type": "choice", "instructions": "x"}),):
try:
router.predict(STATE, {"q": qdef}, model="english")
FAIL.append("rejected/router %s: no error raised" % label)
except ValueError as e:
check_true("rejected/router %s names the question" % label, "'q'" in str(e), str(e))
except Exception as e:
FAIL.append("rejected/router %s: %s instead of ValueError: %s" % (label, type(e).__name__, e))

# every question id is validated, not only the first one put in the dict
try:
agent.system_one(STATE, {"ok": {"type": "noul", "instructions": "Is it urgent?"},
"broken": {"type": "choice", "instructions": "Which team?"}})
FAIL.append("rejected/second question: no error raised")
except ValueError as e:
check_true("rejected/second question names it", "'broken'" in str(e), str(e))
except Exception as e:
FAIL.append("rejected/second question: %s instead of ValueError: %s" % (type(e).__name__, e))

# ...and the shapes that are valid still answer, so this is not validation-only coverage
GOOD = {
"choice": {"type": "choice", "instructions": "Which team?",
"criteria": {"billing": "invoices and refunds", "tech": "bugs"}},
"choice as a list": {"type": "choice", "instructions": "Which team?",
"criteria": ["billing", "tech"]},
"score": {"type": "score", "instructions": "How urgent?",
"criteria": ["no pressure", "soon", "blocking"]},
"noul": {"type": "noul", "instructions": "Does the sender want a reply?"},
"noul with criteria": {"type": "noul", "instructions": "Is it phishing?",
"criteria": {"true": "phishing", "false": "legitimate"}},
"non-string instructions": {"type": "noul", "instructions": {"asks": "for a refund"}},
}
out = agent.system_one(STATE, GOOD)
check("good/one answer per question", sorted(out["answers"]), sorted(GOOD))
check_true("good/choice label", out["answers"]["choice"]["choice"] in ("billing", "tech"),
str(out["answers"]["choice"]))
check_true("good/choice from a list",
out["answers"]["choice as a list"]["choice"] in ("billing", "tech"),
str(out["answers"]["choice as a list"]))
check("good/choice probabilities sum", round(sum(out["answers"]["choice"]["probabilities"].values()), 3), 1.0)
check_true("good/score is in range", 0.0 <= out["answers"]["score"]["score"] <= 2.0,
str(out["answers"]["score"]))
check_true("good/score legend", out["answers"]["score"]["legend"],
{"0": "no pressure", "1": "soon", "2": "blocking"})
check_true("good/noul is a probability", 0.0 <= out["answers"]["noul"]["noul"] <= 1.0,
str(out["answers"]["noul"]))
check_true("good/noul with criteria is a probability",
0.0 <= out["answers"]["noul with criteria"]["noul"] <= 1.0,
str(out["answers"]["noul with criteria"]))
check("good/usage has no output tokens", out["usage"]["output_tokens"], 0)
check_true("good/usage counted input tokens", out["usage"]["input_tokens"] > 0, str(out["usage"]))


print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)
Expand Down