Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions docs/evals.md
Original file line number Diff line number Diff line change
Expand Up @@ -392,3 +392,26 @@ opt in. With `--onnx`, only a bare `--revision <SHA>` applies, to the config and
Drop a JSONL in `research/evals/` and a reviewed baseline beside it, then point a workflow (or
`research/evals/check_regression.py`) at both. The format is the same as the fixture; nothing in
the harness knows about MASSIVE.

## Evidence inspection

`evidence` reads — never writes — the artifacts a fine-tune/eval run already persists and says
what evidence exists, what is missing, and what is insufficient. It loads no model weights and
does not require torch.

```bash
laya-evals evidence --checkpoint ./my-checkpoint [--report report.json]
```

A checkpoint directory must contain `rl_agent_config.json`. Calibration evidence is derived from
the persisted `training.laya_train_calibration` block: a legacy checkpoint with no training
metadata is `UNKNOWN`; a question type with zero items is `MISSING`; a type whose entry records
any upstream `issues` text (not fitted, below `MIN_TYPE_N`/`CALIB_WARN_N`, clamped, unchanged
fit) is `INSUFFICIENT`; a clean fit is `PRESENT`. The helper trusts the persisted `issues`
written by #933 instead of keeping a second numeric threshold source.

The eval report's identity fields (`schema`, `dataset_sha256`, `questions_sha256`, `laya_version`)
are reported with the same semantics as `run`. The checkpoint↔report relationship is claimed only
as a deterministic conflict: if both artifacts expose the same `dataset_sha256`/`questions_sha256`
with different values, it is `INCOMPARABLE`. Everything else is `UNKNOWN` — matching `laya_version`
alone never proves a match, and differing `laya_version` alone never proves a conflict.
21 changes: 20 additions & 1 deletion laya/evals_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import sys
from typing import Any, Dict, List, Optional, Sequence, Tuple

from . import _eval_policy, evals
from . import _eval_policy, evals, evidence
from .evals import EvalError


Expand Down Expand Up @@ -230,6 +230,13 @@ def _build_parser() -> argparse.ArgumentParser:
help="allowed absolute drift; repeatable")
compare.add_argument("--gate-policy", metavar="FILE",
help="apply opt-in per-slice quality rules from a JSON policy")
evidence_cmd = sub.add_parser("evidence",
help="read-only evidence inspection over a checkpoint config and an eval report",
epilog="example: laya-evals evidence --checkpoint ./my-checkpoint --report report.json")
evidence_cmd.add_argument("--checkpoint", required=True,
help="checkpoint directory containing rl_agent_config.json")
evidence_cmd.add_argument("--report", default=None,
help="an existing laya-evals report JSON")

return parser

Expand Down Expand Up @@ -436,6 +443,16 @@ def _cmd_run(args) -> int:
return 0


def _cmd_evidence(args) -> int:
try:
result = evidence.inspect_checkpoint(args.checkpoint, args.report)
except (FileNotFoundError, ValueError) as exc:
print("laya-evals: %s" % exc, file=sys.stderr)
return 2
print(evidence.format_summary(result))
return 0


def _cmd_compare(args) -> int:
# `_identity_of` rather than a bare `config` slice, because a report may carry its identity
# at the top level -- `research/evals/act_head_eval.py` puts `schema` there -- and
Expand Down Expand Up @@ -474,6 +491,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
return _cmd_validate(args)
if args.command == "run":
return _cmd_run(args)
if args.command == "evidence":
return _cmd_evidence(args)
return _cmd_compare(args)
except EvalError as exc:
# A malformed dataset, an unreadable report, or a mistyped pin is a usage error, not a
Expand Down
165 changes: 165 additions & 0 deletions laya/evidence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
"""Read-only evidence inspection over existing Laya artifacts.

This module reports what evidence exists for a checkpoint -- and what is missing,
insufficient, or incomparable -- without loading model weights, without needing
torch, and without writing a new manifest. It only reads artifacts the upstream
train/eval path already persists:

- a checkpoint directory's ``rl_agent_config.json`` (``training.laya_train`` and
``training.laya_train_calibration`` as written by merged #931/#933);
- an existing ``laya-evals`` report JSON (identity semantics from `laya.evals`).

Every state it assigns is either derived directly from an upstream artifact or
marked UNKNOWN. It never manufactures a healthy-looking state from absent input.
"""
from __future__ import annotations

import json
import os
from typing import Any, Dict, List, Optional

PRESENT = "PRESENT"
MISSING = "MISSING"
INSUFFICIENT = "INSUFFICIENT"
UNKNOWN = "UNKNOWN"
INCOMPARABLE = "INCOMPARABLE"

# Documented upstream floors/thresholds. `laya.calibrate.MIN_TYPE_N` and
# `laya.train.CALIB_WARN_N` live in modules that import torch at module scope, so a
# read-only command cannot import them without pulling torch in. We prefer the live
# constants when torch is importable and fall back to the documented values with an
# explicit provenance note when it is not; we never silently duplicate.
def _entry(state: str, reason: str, detail: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
out = {"state": state, "reason": reason}
if detail:
out["detail"] = detail
return out


def inspect_calibration(config: Dict[str, Any]) -> Dict[str, Any]:
"""Evidence state of the persisted calibration fit in a checkpoint config."""
training = config.get("training")
if not isinstance(training, dict):
return _entry(UNKNOWN, "legacy checkpoint: no training metadata persisted; cannot tell whether calibration ran")
calibration = training.get("laya_train_calibration")
if not isinstance(calibration, dict):
return _entry(UNKNOWN, "training metadata present but no calibration block; checkpoint predates #933")
types: Dict[str, Any] = {}
for name, entry in calibration.items():
if not isinstance(entry, dict) or not isinstance(entry.get("items"), int):
types[name] = _entry(UNKNOWN, "calibration entry is malformed")
continue
items = entry["items"]
if items == 0:
types[name] = _entry(MISSING, "question type present but zero calibration items", {"items": 0})
continue
# Persisted issues are the authoritative limitation text (#933 writes
# not-fitted/below-floor/clamped/unchanged-fit there); we do not keep a
# second numeric source of truth.
issues = entry.get("issues")
if not isinstance(issues, list):
types[name] = _entry(UNKNOWN, "calibration entry has no issues list; cannot tell limitations")
elif issues:
types[name] = _entry(INSUFFICIENT, "fitted with recorded limitations: " + "; ".join(str(i) for i in issues),
{"items": items, "issues": issues})
else:
types[name] = _entry(PRESENT, "fitted with no recorded issues on %d items" % items,
{"items": items, "temperature": entry.get("temperature")})
present = sum(1 for t in types.values() if t["state"] == PRESENT)
overall = PRESENT if types and present == len(types) else (
MISSING if types and all(t["state"] == MISSING for t in types.values()) else INSUFFICIENT)
return {
"state": overall,
"types": types,
}


def inspect_report(report: Dict[str, Any]) -> Dict[str, Any]:
"""Evidence state of an eval report's identity fields (reuses EvalReport semantics)."""
config = report.get("config")
if not isinstance(config, dict):
return _entry(UNKNOWN, "report has no config block; identity unknown")
fields = {}
for key in ("schema", "dataset_sha256", "questions_sha256", "laya_version"):
value = config.get(key)
if value is None and report.get(key) is not None:
value = report.get(key)
fields[key] = _entry(PRESENT, "recorded", {"value": value}) if value is not None else _entry(UNKNOWN, "not recorded in this report")
identity_keys = [fields["schema"], fields["dataset_sha256"], fields["questions_sha256"]]
overall = PRESENT if all(f["state"] == PRESENT for f in identity_keys) else UNKNOWN
return {"state": overall, "fields": fields}


_IDENTITY_CONFLICT_KEYS = ("dataset_sha256", "questions_sha256")


def _field(config_like: Any, key: str) -> Any:
if not isinstance(config_like, dict):
return None
if config_like.get(key) is not None:
return config_like.get(key)
nested = config_like.get("training")
if isinstance(nested, dict):
for inner in (nested.get("laya_train"),):
if isinstance(inner, dict) and inner.get(key) is not None:
return inner.get(key)
return None


def relate(config: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:
"""Deterministic checkpoint <-> report relationship.

v1 is deliberately conservative: it emits INCOMPARABLE only when both artifacts
expose the same deterministic identity key with conflicting values, and it never
infers a PRESENT match from `laya_version` (which names the software, not the
checkpoint). Everything else is UNKNOWN -- current artifacts expose no immutable
shared checkpoint identity.
"""
report_config = report.get("config") if isinstance(report.get("config"), dict) else {}
conflicts = []
for key in _IDENTITY_CONFLICT_KEYS:
a = _field(config, key)
b = _field(report_config, key)
if a is not None and b is not None and str(a) != str(b):
conflicts.append(key)
if conflicts:
return _entry(INCOMPARABLE, "checkpoint and report disagree on: " + ", ".join(conflicts))
return _entry(UNKNOWN, "no immutable shared checkpoint identity between this checkpoint artifact and the report")


def inspect_checkpoint(checkpoint_dir: str, report_path: Optional[str] = None) -> Dict[str, Any]:
"""Full read-only evidence inspection for one checkpoint (optionally one report)."""
config_path = os.path.join(checkpoint_dir, "rl_agent_config.json")
if not os.path.isdir(checkpoint_dir) or not os.path.isfile(config_path):
raise FileNotFoundError("no rl_agent_config.json under %r" % checkpoint_dir)
with open(config_path, "r", encoding="utf-8") as handle:
config = json.load(handle)
if not isinstance(config, dict):
raise ValueError("%s is not a JSON object" % config_path)
out: Dict[str, Any] = {"checkpoint": checkpoint_dir, "calibration": inspect_calibration(config)}
if report_path is not None:
with open(report_path, "r", encoding="utf-8") as handle:
report = json.load(handle)
if not isinstance(report, dict):
raise ValueError("%s is not a JSON object" % report_path)
out["report"] = inspect_report(report)
out["relationship"] = relate(config, report)
return out


def format_summary(result: Dict[str, Any]) -> str:
lines: List[str] = ["evidence summary for %s" % result["checkpoint"],
"(reports evidence and limitations found in existing artifacts; this is not a deployment verdict)"]
cal = result["calibration"]
lines.append("calibration: %s" % cal["state"])
for name, t in (cal.get("types") or {}).items():
lines.append(" %-24s %s -- %s" % (name, t["state"], t["reason"]))
if cal.get("state") == UNKNOWN and "reason" in cal:
lines.append(" " + cal["reason"])
if "report" in result:
rep = result["report"]
lines.append("eval report identity: %s" % rep["state"])
for key, f in rep.get("fields", {}).items():
lines.append(" %-18s %s" % (key, f["state"]))
lines.append("checkpoint<->report: %s -- %s" % (result["relationship"]["state"], result["relationship"]["reason"]))
return "\n".join(lines)
1 change: 1 addition & 0 deletions scripts/test_suites.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
"tests/test_conformal_abstention.py",
"tests/test_cli.py",
"tests/test_cli_lang_guess.py",
"tests/test_evidence.py",
"tests/test_mcp.py",
"tests/test_mcp_device.py",
"tests/test_langchain.py",
Expand Down
2 changes: 1 addition & 1 deletion tests/test_evals_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ def _subparsers(parser):


_subs = _subparsers(evals_cli._build_parser())
check("laya-evals subcommands", sorted(_subs), ["compare", "run", "validate"])
check("laya-evals subcommands", sorted(_subs), ["compare", "evidence", "run", "validate"])

_unhelped, _unstated, _options = [], [], {}
for _sub, _parser in sorted(_subs.items()):
Expand Down
138 changes: 138 additions & 0 deletions tests/test_evidence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
"""Contract guard for laya.evidence: readiness evidence states over existing artifacts.

Run: python tests/test_evidence.py
"""
import json
import os
import sys
import tempfile

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from laya import evidence # noqa: E402
from laya import evals_cli # noqa: E402

PASS, FAIL = [], []


def check(name, got, expected):
ok = got == expected
(PASS if ok else FAIL).append(name)
print((" ok " if ok else " FAIL ") + name, "" if ok else "(got %r, expected %r)" % (got, expected))


def check_true(name, condition):
check(name, bool(condition), True)


def calib(config):
return evidence.inspect_calibration(config)


# ------------------------------------------------------ calibration states
legacy = calib({"fine_tuned": True})
check("legacy checkpoint (no training key) -> UNKNOWN", legacy["state"], evidence.UNKNOWN)

legacy2 = calib({"training": {"laya_train": {"base": "x"}}})
check("training but no calibration block -> UNKNOWN", legacy2["state"], evidence.UNKNOWN)

zero = calib({"training": {"laya_train_calibration": {"choice": {"items": 0, "temperature": 1.0, "issues": []}}}})
check("zero items -> MISSING", zero["types"]["choice"]["state"], evidence.MISSING)

low = calib({"training": {"laya_train_calibration": {"choice": {"items": 5, "temperature": 1.0,
"issues": ["not fitted: 5 calibration items, fewer than 10, so the temperature stays 1.0"]}}}})
check("below MIN_TYPE_N -> INSUFFICIENT", low["types"]["choice"]["state"], evidence.INSUFFICIENT)

warned = calib({"training": {"laya_train_calibration": {"choice": {"items": 30, "temperature": 1.2,
"issues": ["fitted on only 30 calibration items"]}}}})
check("fitted below CALIB_WARN_N -> INSUFFICIENT", warned["types"]["choice"]["state"], evidence.INSUFFICIENT)

clamped = calib({"training": {"laya_train_calibration": {"choice": {"items": 200, "temperature": 0.5,
"issues": ["the fitted temperature 0.5 is on the [0.5, 3] clamp"]}}}})
check("clamped fit stays a visible limitation -> INSUFFICIENT", clamped["types"]["choice"]["state"], evidence.INSUFFICIENT)

clean = calib({"training": {"laya_train_calibration": {"choice": {"items": 400, "temperature": 1.1, "issues": []}}}})
check("clean supported fit -> PRESENT", clean["types"]["choice"]["state"], evidence.PRESENT)
check("all-clean overall -> PRESENT", clean["state"], evidence.PRESENT)

mixed = calib({"training": {"laya_train_calibration": {
"choice": {"items": 400, "temperature": 1.1, "issues": []},
"score": {"items": 0, "temperature": 1.0, "issues": []}}}})
check("mixed types overall -> INSUFFICIENT", mixed["state"], evidence.INSUFFICIENT)

malformed = calib({"training": {"laya_train_calibration": {"choice": {"temperature": 1.0}}}})
check("malformed entry -> UNKNOWN", malformed["types"]["choice"]["state"], evidence.UNKNOWN)

# ------------------------------------------------------ report identity
full_report = {"config": {"schema": "laya-evals-report/1", "dataset_sha256": "a",
"questions_sha256": "b", "laya_version": "0.3.28"}, "overall": {}}
rep = evidence.inspect_report(full_report)
check("full identity -> PRESENT", rep["state"], evidence.PRESENT)

old_report = {"config": {"schema": "laya-evals-report/1"}, "overall": {}}
rep2 = evidence.inspect_report(old_report)
check("missing identity fields -> UNKNOWN", rep2["state"], evidence.UNKNOWN)

toplevel = {"schema": "laya-evals-report/1", "dataset_sha256": "a", "questions_sha256": "b",
"config": {}}
check("identity at top level still read", evidence.inspect_report(toplevel)["state"], evidence.PRESENT)

# ------------------------------------------------------ relationship
config = {"training": {"laya_train": {"laya_version": "0.3.28"}}}
match = evidence.relate(config, {"config": {"laya_version": "0.3.28"}})
check("same laya_version alone must NOT prove PRESENT", match["state"], evidence.UNKNOWN)
conflict = evidence.relate(config, {"config": {"laya_version": "0.3.20"}})
check("different laya_version must NOT become INCOMPARABLE", conflict["state"], evidence.UNKNOWN)
conflict_ds = evidence.relate({"dataset_sha256": "a"}, {"config": {"dataset_sha256": "b"}})
check("deterministic identity conflict -> INCOMPARABLE", conflict_ds["state"], evidence.INCOMPARABLE)
unknown = evidence.relate(config, {"config": {}})
check("no shared identity field -> UNKNOWN", unknown["state"], evidence.UNKNOWN)

# ------------------------------------------------------ CLI on a realistic artifact pair
tmp = tempfile.mkdtemp()
ckpt = os.path.join(tmp, "ckpt")
os.makedirs(ckpt)
with open(os.path.join(ckpt, "rl_agent_config.json"), "w", encoding="utf-8") as f:
json.dump({"training": {"laya_train": {"laya_version": "0.3.28"}, "laya_train_calibration": {
"choice": {"items": 400, "temperature": 1.1, "issues": []},
"score": {"items": 30, "temperature": 1.0, "issues": ["fitted on only 30 calibration items"]}}}}, f)
report_path = os.path.join(tmp, "report.json")
with open(report_path, "w", encoding="utf-8") as f:
json.dump({"config": {"schema": "laya-evals-report/1", "dataset_sha256": "a",
"questions_sha256": "b", "laya_version": "0.3.28"}, "overall": {}}, f)

rc = evals_cli.main(["evidence", "--checkpoint", ckpt, "--report", report_path])
check("evidence CLI returns 0 on realistic pair", rc, 0)

rc_bad = evals_cli.main(["evidence", "--checkpoint", os.path.join(tmp, "nope")])
check("missing checkpoint dir -> usage error 2", rc_bad, 2)

# ------------------------------------------------------ no torch required
import subprocess
_BLOCK = (
"import sys, os, json, tempfile\n"
"sys.path.insert(0, os.getcwd())\n"
"class B:\n"
" def find_spec(self, name, path, target=None):\n"
" if name == 'torch' or name.startswith('torch.'):\n"
" raise ImportError('torch blocked')\n"
"sys.meta_path.insert(0, B())\n"
"tmp = tempfile.mkdtemp(); ckpt = os.path.join(tmp, 'c'); os.makedirs(ckpt)\n"
"with open(os.path.join(ckpt, 'rl_agent_config.json'), 'w') as f:\n"
" json.dump({'training': {'laya_train_calibration': {'choice': {'items': 5, 'temperature': 1.0, 'issues': []}}}}, f)\n"
"from laya import evals_cli, evidence\n"
"rc = evals_cli.main(['evidence', '--checkpoint', ckpt])\n"
"res = evidence.inspect_checkpoint(ckpt)\n"
"print('RC', rc, 'TORCH', 'torch' in sys.modules)\n"
)
proc = subprocess.run([sys.executable, "-c", _BLOCK], cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
capture_output=True, text=True, timeout=120)
print(proc.stdout.strip(), proc.stderr.strip()[-300:])
out = proc.stdout
check_true("evidence CLI works with torch blocked", "RC 0" in out)
check_true("torch not imported when blocked", "TORCH False" in out)

if FAIL:
print("\nFAILURES:", FAIL)
sys.exit(1)
print("\nall %d checks passed" % len(PASS))
Loading