diff --git a/laya/agent.py b/laya/agent.py index 15e2eb566..1ea620215 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -251,6 +251,41 @@ def __init__( " See https://pytorch.org/get-started/locally/\n" % (fell_back_from, fell_back_why), flush=True) + @staticmethod + def _check_question(qid: str, qdef: Any) -> None: + """Reject a question that cannot be answered, naming it and what to fix. + + `render_options` reads `criteria` in the shape the question's type expects and the decision + head needs at least one option, so a malformed definition used to surface from three frames + down as something that names neither the question nor the problem: `AttributeError: + 'NoneType' object has no attribute 'items'`, `KeyError: 'bool'`, or a `selected index k out + of range` raised inside the model for a question that ended up with no options at all. + """ + if not isinstance(qdef, dict): + raise ValueError("question %r: definition must be a dict, got %s" + % (qid, type(qdef).__name__)) + t = qdef.get("type") + if t not in QTYPES: + raise ValueError("question %r: unknown type %r; use one of %s" % (qid, t, sorted(QTYPES))) + if "instructions" not in qdef: + raise ValueError("question %r: no 'instructions'; add the text the model should answer" % (qid,)) + crit = qdef.get("criteria") + if t == "choice": + if not isinstance(crit, (dict, list)): + raise ValueError("question %r: a choice question takes 'criteria' as a dict of " + "label -> description, or a list of labels" % (qid,)) + if not crit: + raise ValueError("question %r: a choice question needs at least one criterion" % (qid,)) + elif t == "score": + if not isinstance(crit, list): + raise ValueError("question %r: a score question takes 'criteria' as a list of level " + "descriptions, index 0 first" % (qid,)) + if not crit: + raise ValueError("question %r: a score question needs at least one level" % (qid,)) + elif crit is not None and not isinstance(crit, dict): + raise ValueError("question %r: a noul question takes 'criteria' as a dict with optional " + "'true'/'false' descriptions, or omits it" % (qid,)) + @staticmethod def _to_internal(qdef: Dict) -> Dict: t = qdef["type"] @@ -282,6 +317,7 @@ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[st head_max_len = self.cfg.get("head_max_len", 192) for qid in ids: + self._check_question(qid, questions[qid]) q = self._to_internal(questions[qid]) seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len) if len(markers) != len(render_options(q)): diff --git a/tests/test_criteria.py b/tests/test_criteria.py index 42eb64079..1dd49bb1f 100644 --- a/tests/test_criteria.py +++ b/tests/test_criteria.py @@ -3,6 +3,11 @@ Regression tests for the bug reported in PR #2 (trocker): a `noul` question whose criteria values were dicts raised `TypeError: can only concatenate str (not "dict") to str`, and `choice`/`score` stringified dicts as Python reprs instead of JSON. + +A question whose *shape* is wrong is the other half of the same promise and lives at the bottom of +this file: `Agent.system_one` rejects it by name before anything is rendered or tokenized, instead +of raising `AttributeError: 'NoneType' object has no attribute 'items'` from `render_options` or a +`selected index k out of range` from inside the decision head (#182). """ import json import os @@ -116,6 +121,130 @@ def check_true(name, cond, detail=""): "torch.cuda.is_available() or getattr(torch.version" not in _src) +# --------------------------------------------------------------- malformed question shapes (#182) +# A question that cannot be answered used to fail three frames down, as an exception that named +# neither the question nor the fix: `AttributeError: 'NoneType' object has no attribute 'items'` +# from `render_options` for a `choice` without criteria, `KeyError: 'bool'` from the type table, +# and -- for a question that ended up with no options at all -- a `selected index k out of range` +# raised inside `DecisionModel.forward`, which reads like a bug in laya rather than in the caller's +# definition. The inference path below is real: a tiny from-config encoder, no checkpoint +# downloaded (tests/test_local_e2e.py covers the real weights). +import torch # noqa: E402 +from transformers import AutoConfig, AutoModel # noqa: E402 + +from laya.agent import Agent # noqa: E402 +from laya.common import DecisionModel # noqa: E402 +from laya.router import Router # noqa: E402 + + +class _FakeTok: + """The tokenizer surface `build_sequence` uses, with predictable ids.""" + cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2 + mask_token = "[MASK]" + + def __call__(self, text, add_special_tokens=False): + return {"input_ids": [10 + (len(w) % 90) for w in text.split() if w]} + + +def _tiny_agent(): + """An `Agent` with a tiny random encoder: the inference path, without the download.""" + agent = object.__new__(Agent) + cfg = AutoConfig.for_model("bert", hidden_size=16, num_hidden_layers=1, num_attention_heads=1, + intermediate_size=32, vocab_size=64) + agent.cfg = {"max_len": 256, "head_max_len": 96, "encoder": "tiny"} + agent.tok = _FakeTok() + agent.model = DecisionModel(AutoModel.from_config(cfg), head_layers=1, n_act=2).eval() + agent.device = torch.device("cpu") + agent.dtype = torch.float32 + agent.temperature = [1.0, 1.0, 1.0] + agent.temperature_by_options = {} + return agent + + +STATE = {"body": "I was charged twice for invoice 4411 and want a refund"} +agent = _tiny_agent() +for label, qdef in [ + ("choice without criteria", {"type": "choice", "instructions": "Which team?"}), + ("choice with criteria None", {"type": "choice", "instructions": "Which team?", "criteria": None}), + ("choice with empty criteria", {"type": "choice", "instructions": "Which team?", "criteria": {}}), + ("choice with a tuple of labels", {"type": "choice", "instructions": "Which team?", + "criteria": ("billing", "tech")}), + ("score without criteria", {"type": "score", "instructions": "How urgent?"}), + ("score with an empty list", {"type": "score", "instructions": "How urgent?", "criteria": []}), + ("score with a dict of levels", {"type": "score", "instructions": "How urgent?", + "criteria": {"low": "no pressure", "high": "blocking"}}), + ("noul with list criteria", {"type": "noul", "instructions": "Is it spam?", "criteria": ["a", "b"]}), + ("noul with string criteria", {"type": "noul", "instructions": "Is it spam?", "criteria": "spam?"}), + ("unknown type", {"type": "bool", "instructions": "Is it spam?"}), + ("missing type", {"instructions": "Is it spam?"}), + ("no instructions", {"type": "noul"}), +]: + try: + agent.system_one(STATE, {"q": qdef}) + FAIL.append("rejected/%s: no error raised" % label) + except ValueError as e: + # the message must name the question: a caller with twenty of them needs to know which + check_true("rejected/%s names the question" % label, "'q'" in str(e), str(e)) + check_true("rejected/%s says what to fix" % label, len(str(e)) > 40, str(e)) + except Exception as e: + FAIL.append("rejected/%s: %s instead of ValueError: %s" % (label, type(e).__name__, e)) + +# the same questions through the public entry point, not only the method under it +router = Router() +router.attach("english", agent) +for label, qdef in (("choice without criteria", {"type": "choice", "instructions": "x"}),): + try: + router.predict(STATE, {"q": qdef}, model="english") + FAIL.append("rejected/router %s: no error raised" % label) + except ValueError as e: + check_true("rejected/router %s names the question" % label, "'q'" in str(e), str(e)) + except Exception as e: + FAIL.append("rejected/router %s: %s instead of ValueError: %s" % (label, type(e).__name__, e)) + +# every question id is validated, not only the first one put in the dict +try: + agent.system_one(STATE, {"ok": {"type": "noul", "instructions": "Is it urgent?"}, + "broken": {"type": "choice", "instructions": "Which team?"}}) + FAIL.append("rejected/second question: no error raised") +except ValueError as e: + check_true("rejected/second question names it", "'broken'" in str(e), str(e)) +except Exception as e: + FAIL.append("rejected/second question: %s instead of ValueError: %s" % (type(e).__name__, e)) + +# ...and the shapes that are valid still answer, so this is not validation-only coverage +GOOD = { + "choice": {"type": "choice", "instructions": "Which team?", + "criteria": {"billing": "invoices and refunds", "tech": "bugs"}}, + "choice as a list": {"type": "choice", "instructions": "Which team?", + "criteria": ["billing", "tech"]}, + "score": {"type": "score", "instructions": "How urgent?", + "criteria": ["no pressure", "soon", "blocking"]}, + "noul": {"type": "noul", "instructions": "Does the sender want a reply?"}, + "noul with criteria": {"type": "noul", "instructions": "Is it phishing?", + "criteria": {"true": "phishing", "false": "legitimate"}}, + "non-string instructions": {"type": "noul", "instructions": {"asks": "for a refund"}}, +} +out = agent.system_one(STATE, GOOD) +check("good/one answer per question", sorted(out["answers"]), sorted(GOOD)) +check_true("good/choice label", out["answers"]["choice"]["choice"] in ("billing", "tech"), + str(out["answers"]["choice"])) +check_true("good/choice from a list", + out["answers"]["choice as a list"]["choice"] in ("billing", "tech"), + str(out["answers"]["choice as a list"])) +check("good/choice probabilities sum", round(sum(out["answers"]["choice"]["probabilities"].values()), 3), 1.0) +check_true("good/score is in range", 0.0 <= out["answers"]["score"]["score"] <= 2.0, + str(out["answers"]["score"])) +check_true("good/score legend", out["answers"]["score"]["legend"], + {"0": "no pressure", "1": "soon", "2": "blocking"}) +check_true("good/noul is a probability", 0.0 <= out["answers"]["noul"]["noul"] <= 1.0, + str(out["answers"]["noul"])) +check_true("good/noul with criteria is a probability", + 0.0 <= out["answers"]["noul with criteria"]["noul"] <= 1.0, + str(out["answers"]["noul with criteria"])) +check("good/usage has no output tokens", out["usage"]["output_tokens"], 0) +check_true("good/usage counted input tokens", out["usage"]["input_tokens"] > 0, str(out["usage"])) + + print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) for f in FAIL: print(" FAIL " + f)