Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,31 @@ jobs:
tests/test_truncation_direction.py \
-q

# The `test` job above deliberately stays free of `onnx`: this is the lane that
# installs it, so the exporter's quantization suite runs for real instead of
# skip-guarding into a trivial pass. Weight-free -- it builds tiny synthetic
# graphs, no checkpoint download.
onnx-export:
name: onnx export (quantization, weight-free)
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.11"
cache: pip

- name: Install
run: |
python -m pip install --upgrade pip
pip install torch --index-url https://download.pytorch.org/whl/cpu
pip install -e .
pip install onnx onnxruntime

- name: Quantized-export tests
run: python tests/test_onnx_quantize.py

# The eval harness needs numpy and laya only, so it stays a separate, weight-free job: metric
# math, dataset parsing, the CLI and the API guard. The real-checkpoint gate is the scheduled
# `evals` workflow, which needs weights and network.
Expand Down
17 changes: 17 additions & 0 deletions docs/reference/agent.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,20 @@ exported ONNX model on CPU; import it from `laya.onnx_agent`.
::: laya.agent.load

::: laya.onnx_agent.ONNXAgent

## Quantized export

`scripts/export_onnx.py --quantize` writes an INT8 weight-only quantized copy beside the fp32
export (`laya.onnx` also produces `laya.int8.onnx`). Dynamic per-channel quantization converts
the `MatMul` weights to int8 with activations left in fp32, so no calibration dataset is needed,
and `ONNXAgent` loads the result by pointing `onnx_path` at it. On the English checkpoint, CPU
(M-series, 20 support-ticket states x choice/noul/score): model file 1.6 GB -> 581 MB, p50
per-state latency ~340 ms -> ~250 ms (~1.35x), and zero decision changes versus fp32 (largest single
probability drift 0.09). Per-tensor scales instead of per-channel flipped 3 of 20 states with
drift up to 0.29, which is why the exporter uses per-channel. The int8 graph is CPU-only: ONNX
Runtime has no INT8 MatMul kernel on the CUDAExecutionProvider, and a GPU provider silently
falls back per node.

```bash
python scripts/export_onnx.py --model convaiinnovations/laya --output laya.onnx --quantize
```
49 changes: 48 additions & 1 deletion scripts/export_onnx.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,47 @@

from laya.agent import Agent


def quantize_model(model_path: str, output_path: str) -> str:
"""Write an INT8 weight-only dynamically quantized copy of `model_path`.

Dynamic quantization converts the weights of every `MatMul` (the attention and MLP linear
layers) to int8 while leaving activations in fp32; the quantization scales are computed per
output channel at load time, so no calibration dataset is needed. The graph structure and
the input/output names are unchanged, which is what lets `ONNXAgent` load the result by
pointing `onnx_path` at it. It is CPU-only: ONNX Runtime has no INT8 MatMul kernel on the
CUDAExecutionProvider, so an int8 graph on GPU falls back to CPU.
"""
from onnxruntime.quantization import QuantType, quantize_dynamic

import onnx

model = onnx.load(model_path)
# The torch exporter leaves intermediate `value_info` shapes that disagree with what the
# quantizer's own shape-inference pass re-derives ("Inferred shape and existing shape
# differ"). The declarations are informational only, so drop them and let quantization
# recompute whatever it needs.
del model.graph.value_info[:]
quantize_dynamic(
model_input=model,
model_output=output_path,
op_types_to_quantize=["MatMul"],
weight_type=QuantType.QInt8,
# One scale per output channel rather than one per tensor. On the English checkpoint
# measured on 20 support-ticket states x choice/noul/score, per-tensor int8 flipped 3
# of 20 decisions (max probability drift 0.29); per-channel flipped none (max 0.09)
# at the same size and speed.
per_channel=True,
)
return output_path


def int8_output_path(output_path: str) -> str:
"""`laya.onnx` -> `laya.int8.onnx`, next to the fp32 export it was quantized from."""
root, ext = os.path.splitext(output_path)
return "%s.int8%s" % (root, ext or ".onnx")


def export_to_onnx(model_id_or_path: str, output_path: str):
print(f"Loading PyTorch Agent from: {model_id_or_path}")
agent = Agent(model_id_or_path, compile=False, device="cpu")
Expand Down Expand Up @@ -78,6 +119,12 @@ def export_to_onnx(model_id_or_path: str, output_path: str):
parser = argparse.ArgumentParser(description="Export a Laya model to ONNX format")
parser.add_argument("--model", type=str, default="convaiinnovations/laya", help="HuggingFace Hub ID or local path")
parser.add_argument("--output", type=str, default="laya.onnx", help="Output path for the ONNX file")
parser.add_argument("--quantize", action="store_true",
help="Also write an INT8 weight-only quantized copy (CPU-only speed and "
"size win) next to --output, named <output>.int8.onnx")
args = parser.parse_args()

export_to_onnx(args.model, args.output)
if args.quantize:
int8_path = quantize_model(args.output, int8_output_path(args.output))
print(f"Successfully wrote INT8 quantized model to: {int8_path}")
140 changes: 140 additions & 0 deletions tests/test_onnx_quantize.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
"""INT8 export: `scripts/export_onnx.py --quantize` writes a weight-only quantized copy.

`ONNXAgent` loads any graph that keeps the fp32 export's input/output names, so the whole
feature is one function: `quantize_model` (dynamic per-channel INT8 MatMul weights via
onnxruntime) plus
`int8_output_path` (the sidecar naming) and the CLI flag. No checkpoint and no torch model is
downloaded here: the quantizer is exercised on a tiny hand-built MatMul graph, which is exactly
the shape it targets in the real export, and the CLI is checked through `--help`.

Run: python tests/test_onnx_quantize.py
"""
import importlib.util
import os
import subprocess
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import numpy as np # noqa: E402
import onnx # noqa: E402
from onnx import TensorProto, helper, numpy_helper # noqa: E402

_here = os.path.dirname(os.path.abspath(__file__))
_spec = importlib.util.spec_from_file_location("export_onnx",
os.path.join(_here, os.pardir, "scripts", "export_onnx.py"))
export_onnx = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(export_onnx)

PASS, FAIL = [], []


def check(name, got, want):
if got == want:
PASS.append(name)
else:
FAIL.append("%s:\n got %r\n want %r" % (name, got, want))


def check_true(name, cond, detail=""):
if cond:
PASS.append(name)
else:
FAIL.append("%s %s" % (name, detail))


# ---------------------------------------------------------------- a tiny fp32 MatMul graph
def _build_model(path):
rng = np.random.default_rng(7)
x = rng.normal(size=(1, 8)).astype(np.float32)
w = (np.eye(8, dtype=np.float32) + 0.02 * rng.normal(size=(8, 8)).astype(np.float32))
model = helper.make_model(
helper.make_graph(
[helper.make_node("MatMul", ["input_ids", "weight"], ["logits"])],
"mini",
[helper.make_tensor_value_info("input_ids", TensorProto.FLOAT, [1, 8])],
[helper.make_tensor_value_info("logits", TensorProto.FLOAT, [1, 8])],
[numpy_helper.from_array(w, name="weight")],
),
opset_imports=[helper.make_opsetid("", 18)],
)
model.ir_version = 13 # keep the graph loadable by the installed onnxruntime
onnx.save(model, path)
return x


tmp = os.path.join(_here, "_tmp_onnx_quantize")
os.makedirs(tmp, exist_ok=True)
fp32_path = os.path.join(tmp, "mini.onnx")
int8_path = os.path.join(tmp, "mini.int8.onnx")
x = _build_model(fp32_path)

# ---------------------------------------------------------------- int8_output_path
check("naming/laya.onnx -> laya.int8.onnx", export_onnx.int8_output_path("laya.onnx"),
"laya.int8.onnx")
check("naming/keeps the directory", export_onnx.int8_output_path("/models/laya.onnx"),
"/models/laya.int8.onnx")
check("naming/extensionless input still gets .onnx", export_onnx.int8_output_path("laya"),
"laya.int8.onnx")

# ---------------------------------------------------------------- quantize_model
out = export_onnx.quantize_model(fp32_path, int8_path)
check("quantize/returns the output path", out, int8_path)
check_true("quantize/writes the file", os.path.exists(int8_path))

q_model = onnx.load(int8_path)
int8_tensors = [t for t in q_model.graph.initializer if t.data_type == TensorProto.INT8]
check_true("quantize/MatMul weight is stored as INT8", len(int8_tensors) >= 1,
[(t.name, t.data_type) for t in q_model.graph.initializer])
scales = [t for t in q_model.graph.initializer if t.name.endswith("_scale")]
check_true("quantize/scales are per output channel (per-tensor flips decisions on the real model)",
scales and all(numpy_helper.to_array(t).size > 1 for t in scales),
[(t.name, list(t.dims)) for t in scales])
check("quantize/graph keeps the I/O names ONNXAgent binds to",
([i.name for i in q_model.graph.input], [o.name for o in q_model.graph.output]),
(["input_ids"], ["logits"]))

import onnxruntime as ort # noqa: E402

sess = ort.InferenceSession(int8_path, providers=["CPUExecutionProvider"])
got = sess.run(["logits"], {"input_ids": x})[0]
fp32_sess = ort.InferenceSession(fp32_path, providers=["CPUExecutionProvider"])
ref = fp32_sess.run(["logits"], {"input_ids": x})[0]
check_true("quantize/outputs stay close to fp32 on a well-conditioned weight",
np.allclose(got, ref, rtol=0.05, atol=0.05),
"max abs diff %.4f" % float(np.max(np.abs(got - ref))))
check_true("quantize/magnitude is actually exercised (not an empty pass)",
float(np.max(np.abs(got - ref))) > 0.0)

def _weight_bytes(model):
"""Raw bytes held by initializers -- the honest size comparison on a tiny graph,
where container overhead dwarfs an 8x8 weight."""
return sum(numpy_helper.to_array(t).nbytes for t in model.graph.initializer)


check_true("quantize/weight storage shrinks vs fp32",
_weight_bytes(q_model) < _weight_bytes(onnx.load(fp32_path)),
(_weight_bytes(q_model), _weight_bytes(onnx.load(fp32_path))))

# the fp32 export itself is untouched -- the quantized model is a sidecar, so A/B is possible
check("quantize/fp32 model still loads and matches itself",
np.allclose(fp32_sess.run(["logits"], {"input_ids": x})[0], ref, atol=0), True)

# ---------------------------------------------------------------- CLI surface
help_text = subprocess.run(
[sys.executable, os.path.join(_here, os.pardir, "scripts", "export_onnx.py"), "--help"],
capture_output=True, text=True, timeout=120,
).stdout
check_true("cli/--quantize is documented in --help", "--quantize" in help_text,
[ln for ln in help_text.splitlines() if "quantize" in ln])
check_true("cli/help names the sidecar output", ".int8.onnx" in help_text)


# ---------------------------------------------------------------- report
import shutil # noqa: E402

shutil.rmtree(tmp, ignore_errors=True)
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)
sys.exit(1 if FAIL else 0)
Loading