diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c996a085..c1bc352fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,7 @@ jobs: python tests/test_decision_model.py python tests/test_head_checkpointing.py python tests/test_packaging.py + python tests/test_doc_tables.py python tests/test_tokenizer_cache.py python tests/test_lazy_import.py python tests/test_lang_guess.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f89264f0e..208613a37 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,6 +45,7 @@ jobs: python tests/test_shortlist.py python tests/test_decision_model.py python tests/test_packaging.py + python tests/test_doc_tables.py python tests/test_lang_guess.py python tests/test_calibration_persistence.py python tests/test_context_manager.py diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 095dda6c1..ea7fa1c4c 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -277,16 +277,20 @@ can be re-checked without a GPU. | checkpoint | type | n | max \|p_fast - p_stock\| | max \|p_fast - p_fp32\| | max \|p_stock - p_fp32\| | argmax fast = stock | fast = fp32 | |---|---|---|---|---|---|---|---| | laya | choice | 48 | 0.031 | **0.022** | 0.024 | 47/48 | 47/48 | -| laya | noul | 180 | 0.076 | **0.044** | 0.058 | 180/180 | 180/180 | +| laya | noul | 180 | 0.076 | **0.043** | 0.058 | 180/180 | 180/180 | | laya | score | 60 | 0.015 | **0.011** | 0.017 | 59/60 | 60/60 | | laya-multilingual | choice | 48 | 0.049 | **0.015** | 0.039 | 47/48 | 47/48 | -| laya-multilingual | noul | 180 | 0.037 | 0.046 | 0.045 | 180/180 | 179/180 | +| laya-multilingual | noul | 180 | 0.037 | 0.045 | 0.045 | 180/180 | 179/180 | | laya-multilingual | score | 60 | 0.010 | **0.009** | 0.009 | 59/60 | 59/60 | -The fast path is at least as close to the fp32 reference as the stock bf16 path is (the residual stream stays in fp32 in -both), and the two bf16 paths differ from each other only by bf16 accumulation order; the few argmax disagreements are -near-tie options, and on every one of them the fast path agrees with fp32. Dataset accuracy / ECE (AG News, dair-ai -emotion, 1,000 samples each) are identical within noise; see `benchmarks/bench_fast.py --eval 1000`. +The fast path stays close to the fp32 reference on every row — at most **0.046** away, against 0.058 for the stock +path on the same row — and no row is more than **0.076** from stock. The residual stream stays in fp32 in both, and +the two bf16 paths differ from each other only by bf16 accumulation order; the few argmax disagreements are near-tie +options, and on every one of them the fast path agrees with fp32. One row is the exception to the stronger reading +that used to be printed here: on `laya-multilingual` `noul` the stock bf16 path is marginally closer to fp32 than the +fast path is (0.0446 against 0.0455), so this table does not show that the fast path is never further from fp32. +Dataset accuracy / ECE (AG News, dair-ai emotion, 1,000 samples each) are identical within noise; see +`benchmarks/bench_fast.py --eval 1000`. ### Latency, `agent.predict()` end to end (ms, incl. tokenization) diff --git a/README.md b/README.md index f0c83e51f..919731c91 100644 --- a/README.md +++ b/README.md @@ -485,8 +485,8 @@ agent = laya.load("convaiinnovations/laya", fast=True) # or: agent.accelerate( agent.predict(state, questions) # same API, same answers ``` -Numerics: on a fixed set of 60 states the fast path is at least as close to an fp32 forward as the stock bf16 -path is (max |Δp| ≤ 0.05 vs fp32 on both checkpoints, argmax agreement ≥ 47/48 per question type; every per-option +Numerics: on a fixed set of 60 states the fast path stays within 0.046 of an fp32 forward and within 0.076 of the stock +bf16 path (max |Δp| ≤ 0.05 vs fp32 on both checkpoints, argmax agreement ≥ 47/48 per question type; every per-option probability is in `benchmarks/results/parity_*.json`) — see `benchmarks/parity_fast.py` and [BENCHMARKS.md](BENCHMARKS.md#gpu-fast-path). Falls back to the stock forward on CPU/MPS or when `tilelang` is not installed; `agent.deaccelerate()` restores it. Kernels compile once per shape bucket on first use (a few seconds, cached on disk). diff --git a/tests/test_doc_tables.py b/tests/test_doc_tables.py new file mode 100644 index 000000000..f57677caf --- /dev/null +++ b/tests/test_doc_tables.py @@ -0,0 +1,178 @@ +"""The benchmark tables that cite a committed JSON must still equal it (#298). + +`BENCHMARKS.md:274` promises the fast-path comparison "can be re-checked without a GPU" +because every per-option probability is in `benchmarks/results/parity_*.json`. Two cells +did not match it, and the sentence under the table claimed the fast path is never further +from fp32 than the stock path is, which the `laya-multilingual` `noul` row contradicts. + +Re-measuring would not have caught either: the run is not stale, the transcription is. So +this reads the published table and the committed JSON and compares them, which is +deterministic and needs no weights, no GPU and no network. + +Only tables with a committed artifact are checked. Everything that cannot be checked from +this repository is listed at the end rather than silently skipped, so the gap stays visible. +""" +from __future__ import annotations + +import json +import os +import re +import sys +from typing import Any, Dict, List, Optional, Tuple + +PASS: List[str] = [] +FAIL: List[str] = [] + + +def check(what: str, got: Any, want: Any) -> None: + if got == want: + PASS.append(what) + else: + FAIL.append("%s: got %r, want %r" % (what, got, want)) + + +def check_true(what: str, cond: bool, detail: Any = "") -> None: + if cond: + PASS.append(what) + else: + FAIL.append("%s: %s" % (what, detail)) + + +def read(path: str) -> str: + # newline="" so a CRLF checkout and an LF checkout parse identically + with open(path, encoding="utf-8", newline="") as fh: + return fh.read().replace("\r\n", "\n") + + +def rounded(value: float, places: int = 3) -> str: + """Format the way a person transcribing a run would: nearest, halves away from zero. + + `round()` is not usable here. It rounds halves to even, so `round(0.0005, 3)` is `0.0` + and `round(0.0095, 3)` is `0.009`, either of which would report a correct cell as + drifted. The table's own `0.045` from `0.045488` and `0.010` from `0.009501` show the + convention actually used is the ordinary one. + """ + from decimal import ROUND_HALF_UP, Decimal + + quantum = Decimal(1).scaleb(-places) + return str(Decimal(repr(value)).quantize(quantum, rounding=ROUND_HALF_UP)) + + +# --------------------------------------------------------------- the published parity table +BENCHMARKS = "BENCHMARKS.md" +PARITY_DIR = os.path.join("benchmarks", "results") + +# checkpoint -> the file its row cites, and the row's own name for the model +SOURCES = { + "laya": "parity_english_rtx4070.json", + "laya-multilingual": "parity_multilingual_rtx4070.json", +} + +HEADER = ("| checkpoint | type | n | max \\|p_fast - p_stock\\| | max \\|p_fast - p_fp32\\| " + "| max \\|p_stock - p_fp32\\| | argmax fast = stock | fast = fp32 |") + + +def split_row(line: str) -> List[str]: + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def unstyled(cell: str) -> str: + """The table bolds the columns it wants read first; the value is what matters.""" + return cell.replace("**", "").replace("\\|", "|").strip() + + +def parse_table(text: str, header: str) -> List[List[str]]: + lines = text.split("\n") + try: + start = lines.index(header) + except ValueError: + return [] + rows = [] + for line in lines[start + 2:]: # skip the |---| separator + if not line.startswith("|"): + break + rows.append(split_row(line)) + return rows + + +def main() -> int: + for path in (BENCHMARKS, PARITY_DIR): + check_true("%s exists" % path, os.path.exists(path), "missing") + + rows = parse_table(read(BENCHMARKS), HEADER) + check_true("parity table/parsed six rows", len(rows) == 6, "got %d" % len(rows)) + check_true("parity table/every row has eight cells", + all(len(r) == 8 for r in rows), [len(r) for r in rows]) + + summaries: Dict[str, Dict[str, Any]] = {} + for ckpt, name in SOURCES.items(): + with open(os.path.join(PARITY_DIR, name), encoding="utf-8") as fh: + summaries[ckpt] = json.load(fh)["summary"] + + cols = ("d_fast_stock", "d_fast_fp32", "d_stock_fp32") + for row in rows: + ckpt, qtype = row[0], row[1] + if ckpt not in summaries: + FAIL.append("parity table/unknown checkpoint %r" % ckpt) + continue + summary = summaries[ckpt].get(qtype) + if summary is None: + FAIL.append("parity table/%s has no %r summary in its JSON" % (ckpt, qtype)) + continue + + label = "%s/%s" % (ckpt, qtype) + check("parity table/%s n" % label, int(row[2]), summary["n"]) + for cell, key in zip(row[3:6], cols): + check("parity table/%s %s" % (label, key), unstyled(cell), rounded(summary[key])) + for cell, key in zip(row[6:8], ("agree_fast_stock", "agree_fast_fp32")): + check("parity table/%s %s" % (label, key), unstyled(cell), + "%d/%d" % (summary[key], summary["n"])) + + # ------------------------------------------------- the sentence under the table + # BENCHMARKS.md and README.md both asserted the fast path is never further from fp32 than + # the stock bf16 path is. That is checkable from the same JSON, and it was false on one + # row, so the sentence now states the bound the data actually supports instead. What is + # asserted here is that bound, and that the row which falsified the old claim is still + # the one the new wording accounts for. + for ckpt, summary in summaries.items(): + for qtype, s in summary.items(): + if not isinstance(s, dict): + continue + worst = max(s["d_fast_fp32"], s["d_stock_fp32"], s["d_fast_stock"]) + check_true("parity bound/%s/%s within the stated 0.076" % (ckpt, qtype), + worst <= 0.076 + 5e-4, worst) + # the per-row agreement counts are checked above against the table itself; this + # pins the floor the prose quotes, which is the worst row in the set (47/48) + check_true("parity bound/%s/%s agreement floor" % (ckpt, qtype), + min(s["agree_fast_fp32"], s["agree_fast_stock"]) >= 47, + (s["agree_fast_fp32"], s["agree_fast_stock"])) + + # the old wording was "at least as close to fp32 as the stock bf16 path is"; this records + # that it is not universally true, so a future edit cannot quietly restore it + contradicted = [(c, q) for c, s in summaries.items() for q, v in s.items() + if isinstance(v, dict) and v["d_fast_fp32"] > v["d_stock_fp32"]] + check("the row that falsified the old sentence is still the multilingual noul one", + contradicted, [("laya-multilingual", "noul")]) + + # ------------------------------------------------- what this cannot check + unbacked = [ + ("README.md typed-decisions rows", "no committed artifact for the fine-tuned checkpoint"), + ("README.md / BENCHMARKS.md 103-332 q/s throughput", "hand-picked from a range spanning 55-553"), + ("BENCHMARKS.md fast-path latency table", "the committed fast_*.json have no 'eval' key"), + ("README.md CPU/T4 reload latencies", "research/latency_benchmark_results.json is absent"), + ] + check_true("unbacked tables are reported, not skipped", len(unbacked) == 4, unbacked) + + print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) + for f in FAIL: + print(" FAIL " + f) + if not FAIL: + print("every cell of the parity table matches its committed JSON") + print("not checkable from this repository (not asserted either way):") + for name, why in unbacked: + print(" - %s: %s" % (name, why)) + return 1 if FAIL else 0 + + +if __name__ == "__main__": + sys.exit(main())