diff --git a/laya/agent.py b/laya/agent.py index 7a1ad6fa4..7538ef346 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -145,6 +145,7 @@ def __init__( token: Optional[str] = None, subfolder: Optional[str] = None, fast: bool = False, + compile: bool = False, ): """Load a Laya checkpoint. @@ -248,11 +249,15 @@ def __init__( # ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder. # That is a loss for the batch sizes Laya runs (a handful of questions per call) and can - # hang on some platforms, so keep the eager path. + # hang on some platforms, so keep the eager path unless explicitly requested. try: - self.model.encoder.config.reference_compile = False + self.model.encoder.config.reference_compile = compile except Exception: pass + + # The TileLang fast path replaces the forward itself, so it takes precedence over compile. + if compile and not fast: + self.model = torch.compile(self.model) # Keep what the checkpoint shipped for inspection, but only ever apply clamped values: # some buckets are fitted to sharpen rather than soften (see clamp_temperature). diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py new file mode 100644 index 000000000..1f15444d8 --- /dev/null +++ b/laya/onnx_agent.py @@ -0,0 +1,199 @@ +import json +import os +import warnings +from typing import Any, Dict, Optional, Union + +import numpy as np + +from laya.common import ( + QTYPES, + build_sequence, + collate_items, + confidence_from_probs, + render_options, + temp_bucket, + TEMP_MIN, + TEMP_MAX, + clamp_temperature, +) + + +class ONNXAgent: + """System 1 decision model runtime via ONNX: fast CPU-optimized decisions.""" + + def __init__( + self, + model_id_or_path: str, + onnx_path: str = "laya.onnx", + subfolder: Optional[str] = None, + ): + """Load a Laya agent backed by ONNX Runtime. + + Args: + model_id_or_path: HuggingFace Hub ID or local path to the original PyTorch checkpoint + (used to load the tokenizer and config). + onnx_path: Path to the exported .onnx file. + subfolder: Optional subfolder if downloading from a repo bundle. + """ + import onnxruntime as ort + from transformers import AutoTokenizer + + model_dir = model_id_or_path + if not os.path.exists(model_dir): + if model_id_or_path.startswith(("/", "./", "../")) or os.path.isabs(model_id_or_path): + raise FileNotFoundError( + f"Local model path not found: {model_id_or_path!r}." + ) + from huggingface_hub import snapshot_download + + prefix = f"{subfolder}/" if subfolder else "" + kw = { + "allow_patterns": [prefix + name for name in ( + "rl_agent_config.json", "tokenizer/*", "encoder/*", + )], + } + model_dir = snapshot_download(model_id_or_path, **kw) + + if subfolder: + model_dir = os.path.join(model_dir, subfolder) + if not os.path.isdir(model_dir): + raise FileNotFoundError( + f"Subfolder {subfolder!r} not found in {model_id_or_path!r}." + ) + + cfg_path = os.path.join(model_dir, "rl_agent_config.json") + if not os.path.exists(cfg_path): + raise FileNotFoundError( + f"Incompatible model: {model_id_or_path!r} does not contain 'rl_agent_config.json'." + ) + + with open(cfg_path) as f: + self.cfg = json.load(f) + + if not os.path.exists(onnx_path): + raise FileNotFoundError( + f"ONNX model not found at {onnx_path!r}. Please run export_onnx.py first." + ) + + # Load Tokenizer + tok_dir = os.path.join(model_dir, "tokenizer") + self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder")) + + # Initialize ONNX Runtime Session (auto-detect GPU if available) + available = ort.get_available_providers() + providers = ( + ["CUDAExecutionProvider", "CPUExecutionProvider"] + if "CUDAExecutionProvider" in available + else ["CPUExecutionProvider"] + ) + self.session = ort.InferenceSession(onnx_path, providers=providers) + + self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0]) + self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {}) + self.temperature = [clamp_temperature(t) for t in self.temperature_raw] + self.temperature_by_options = {k: clamp_temperature(v) + for k, v in self.temperature_by_options_raw.items()} + rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items() + if clamp_temperature(v) != float(v)] + rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw) + if clamp_temperature(t) != float(t)] + if rejected: + warnings.warn( + "laya ONNX: this checkpoint ships temperatures outside [%g, %g] which would distort " + "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated." + % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)), + RuntimeWarning, stacklevel=2) + + @staticmethod + def _to_internal(qdef: Dict) -> Dict: + t = qdef["type"] + crit = qdef.get("criteria") + if t == "choice" and isinstance(crit, list): + crit = {c: None for c in crit} + ins = qdef["instructions"] + if not isinstance(ins, str): + ins = json.dumps(ins) + return {"t": t, "ins": ins, "crit": crit} + + def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + ids = list(questions.keys()) + items = [] + max_len = self.cfg.get("max_len", 512) + head_max_len = self.cfg.get("head_max_len", 192) + + for qid in ids: + q = self._to_internal(questions[qid]) + seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len) + if len(markers) != len(render_options(q)): + raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len)) + items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]}) + + b = collate_items([items], self.tok.pad_token_id) + + # Prepare ONNX inputs as numpy arrays + ort_inputs = { + "input_ids": b["input_ids"].numpy().astype(np.int64), + "attention_mask": b["attention_mask"].numpy().astype(np.int64), + "marker_pos": b["marker_pos"].numpy().astype(np.int64), + "marker_mask": b["marker_mask"].numpy().astype(bool), + "qtype": b["qtype"].numpy().astype(np.int64), + } + + # Run ONNX inference + ort_outs = self.session.run(["logits", "act_logits"], ort_inputs) + logits = ort_outs[0] + act_logits = ort_outs[1] + + # Compute softmax for actions manually in numpy + act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True)) + act = act_exp / np.sum(act_exp, axis=-1, keepdims=True) + + answers = {} + n_tokens = int(b["attention_mask"].sum()) + + for r, qid in enumerate(ids): + q = self._to_internal(questions[qid]) + k = len(items[r]["markers"]) + qt = QTYPES[q["t"]] + t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt]) + z = logits[r, :k] / t_scale + p = np.exp(z - z.max()) + p = p / p.sum() + + conf_score = round(confidence_from_probs(p, k), 4) + ext = {"act_probability": round(float(act[r, 0]), 4)} + + if q["t"] == "choice": + keys = list(q["crit"].keys()) + answers[qid] = { + "type": "choice", + "choice": keys[int(p.argmax())], + "probabilities": {kk: round(float(v), 4) for kk, v in zip(keys, p)}, + "confidence": conf_score, + "action": ext, + } + elif q["t"] == "score": + exp_score = float((np.arange(k) * p).sum()) + answers[qid] = { + "type": "score", + "score": round(exp_score, 4), + "legend": {str(i): c for i, c in enumerate(q["crit"])}, + "probabilities": {str(i): round(float(v), 4) for i, v in enumerate(p)}, + "confidence": conf_score, + "action": ext, + } + else: + answers[qid] = { + "type": "noul", + "noul": round(float(p[1]), 4), + "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4), + "action": ext, + } + + return { + "model": "laya-rl-agent-onnx", + "answers": answers, + "usage": {"input_tokens": n_tokens, "output_tokens": 0}, + } + + predict = system_one diff --git a/pyproject.toml b/pyproject.toml index 092e89315..1eed70f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,8 @@ fast = ["tilelang>=0.1.14"] # `pip install laya[mcp]` adds the MCP stdio server (laya.mcp / the laya-mcp-server CLI). mcp = ["mcp>=2.2.0"] +onnx = ["onnx", "onnxruntime"] + [project.scripts] laya = "laya.cli:main" laya-serve = "laya.serve:main" diff --git a/scripts/export_onnx.py b/scripts/export_onnx.py new file mode 100644 index 000000000..b48d233ac --- /dev/null +++ b/scripts/export_onnx.py @@ -0,0 +1,83 @@ +import argparse +import os +import torch + +from laya.agent import Agent + +def export_to_onnx(model_id_or_path: str, output_path: str): + print(f"Loading PyTorch Agent from: {model_id_or_path}") + agent = Agent(model_id_or_path, compile=False, device="cpu") + + print("Creating dummy input tensors...") + # 1. Dummy tensors for tracing + # (batch_size=1, seq_len=16) + dummy_input_ids = torch.randint(0, 100, (1, 16), dtype=torch.long) + dummy_attention_mask = torch.ones((1, 16), dtype=torch.long) + + # (batch_size=1, num_markers=2) + dummy_marker_pos = torch.tensor([[1, 5]], dtype=torch.long) + dummy_marker_mask = torch.tensor([[True, True]], dtype=torch.bool) + + # (batch_size=1) + dummy_qtype = torch.tensor([0], dtype=torch.long) + + inputs = ( + dummy_input_ids, + dummy_attention_mask, + dummy_marker_pos, + dummy_marker_mask, + dummy_qtype, + ) + + # 2. Define dynamic axes so the model can accept variable batch sizes and sequence lengths + dynamic_axes = { + "input_ids": {0: "batch_size", 1: "seq_len"}, + "attention_mask": {0: "batch_size", 1: "seq_len"}, + "marker_pos": {0: "batch_size", 1: "num_markers"}, + "marker_mask": {0: "batch_size", 1: "num_markers"}, + "qtype": {0: "batch_size"}, + "logits": {0: "batch_size", 1: "num_markers"}, + "act_logits": {0: "batch_size"}, + } + + input_names = [ + "input_ids", + "attention_mask", + "marker_pos", + "marker_mask", + "qtype", + ] + + output_names = ["logits", "act_logits"] + + print(f"Exporting to {output_path} (this may take a minute)...") + + out_dir = os.path.dirname(os.path.abspath(output_path)) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + + # We must detach the encoder because ONNX export runs the model in trace mode. + # The `detach_encoder` flag in forward() just detaches the hidden state gradient, + # but we don't even need to pass it since kwargs are ignored by tracing. + + torch.onnx.export( + agent.model, + inputs, + output_path, + export_params=True, + opset_version=18, + do_constant_folding=True, + input_names=input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + ) + + print(f"Successfully exported ONNX model to: {output_path}") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Export a Laya model to ONNX format") + parser.add_argument("--model", type=str, default="convaiinnovations/laya", help="HuggingFace Hub ID or local path") + parser.add_argument("--output", type=str, default="laya.onnx", help="Output path for the ONNX file") + args = parser.parse_args() + + export_to_onnx(args.model, args.output) diff --git a/tests/test_onnx.py b/tests/test_onnx.py new file mode 100644 index 000000000..5f35b59fd --- /dev/null +++ b/tests/test_onnx.py @@ -0,0 +1,91 @@ +import os +import sys +import tempfile +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from laya.agent import Agent + +# Skip the test if onnx isn't installed +try: + import onnxruntime + from laya.onnx_agent import ONNXAgent + from scripts.export_onnx import export_to_onnx + HAS_ONNX = True +except ImportError: + HAS_ONNX = False + +import pytest + +@pytest.mark.skipif(not HAS_ONNX, reason="onnx and onnxruntime are required") +def test_onnx_numerical_parity(): + """Verify that PyTorch and ONNX agents produce identical outputs for all 3 question types.""" + model_id = "convaiinnovations/laya" + + with tempfile.TemporaryDirectory() as tmpdir: + onnx_path = os.path.join(tmpdir, "laya.onnx") + + # 1. Export to ONNX + export_to_onnx(model_id, onnx_path) + + # 2. Load both PyTorch and ONNX Agents + agent_pt = Agent(model_id, compile=False, device="cpu") + agent_onnx = ONNXAgent(model_id, onnx_path) + + # 3. Define a state and all 3 question types + state = {"request": "Refactor this service using dependency injection"} + questions = { + "intent": { + "type": "choice", + "instructions": "What is the user asking to do?", + "criteria": { + "refactor": "code refactoring, rewriting", + "bug_fix": "fixing bugs, issues", + "feature": "adding new features" + } + }, + "complexity": { + "type": "score", + "instructions": "How complex is this request?", + "criteria": ["trivial", "simple", "moderate", "complex", "very complex"] + }, + "safety": { + "type": "noul", + "instructions": "Does this request involve any unsafe or harmful content?" + }, + } + + # 4. Predict with both agents + res_pt = agent_pt.predict(state, questions) + res_onnx = agent_onnx.predict(state, questions) + + # 5. Assert parity for choice question + assert res_pt["answers"]["intent"]["choice"] == res_onnx["answers"]["intent"]["choice"], \ + f"Choice mismatch: {res_pt['answers']['intent']['choice']} vs {res_onnx['answers']['intent']['choice']}" + + probs_pt = res_pt["answers"]["intent"]["probabilities"] + probs_onnx = res_onnx["answers"]["intent"]["probabilities"] + for k in probs_pt.keys(): + np.testing.assert_allclose(probs_pt[k], probs_onnx[k], atol=1e-3, rtol=1e-3) + + # 6. Assert parity for score question + np.testing.assert_allclose( + res_pt["answers"]["complexity"]["score"], + res_onnx["answers"]["complexity"]["score"], + atol=1e-3, rtol=1e-3, + ) + + probs_pt_s = res_pt["answers"]["complexity"]["probabilities"] + probs_onnx_s = res_onnx["answers"]["complexity"]["probabilities"] + for k in probs_pt_s.keys(): + np.testing.assert_allclose(probs_pt_s[k], probs_onnx_s[k], atol=1e-3, rtol=1e-3) + + # 7. Assert parity for noul question + np.testing.assert_allclose( + res_pt["answers"]["safety"]["noul"], + res_onnx["answers"]["safety"]["noul"], + atol=1e-3, rtol=1e-3, + ) + + print("ONNX numerical parity test passed for all 3 question types!")