Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions laya/_eval_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,19 @@ def _finite_number(value: Any) -> bool:


def load_policy(path: str) -> Dict[str, Any]:
"""Read and validate a small, versioned JSON gate before a checkpoint is loaded."""
"""Read and validate a small, versioned JSON gate before a checkpoint is loaded.

A `RecursionError` is one of the ways a document cannot be read: `json` reports a nesting depth
it will not walk as a stack overflow, not as a `ValueError`. Leaving it out here made a policy
too deep to parse escape `laya-evals`' usage-error handler as a traceback with exit 1 -- the
code the CLI keeps for "a threshold or a tolerance failed" -- which `docs/evals.md` says may
only be exit 2, before a checkpoint loads. `_load_report` in `laya/evals_cli.py` already reads
a caller's JSON this way.
"""
try:
with open(path, "r", encoding="utf-8") as handle:
policy = json.load(handle)
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
except (OSError, UnicodeError, json.JSONDecodeError, RecursionError) as exc:
raise evals.EvalError("invalid gate policy %r: %s" % (path, exc)) from exc
if not isinstance(policy, dict) or set(policy) != {"version", "rules"}:
raise evals.EvalError("gate policy needs exactly 'version' and 'rules'")
Expand Down
22 changes: 20 additions & 2 deletions tests/test_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -3454,17 +3454,35 @@ def test_cli_relative_slice_rule_requires_baseline_before_loading(monkeypatch, t
assert not built


def test_cli_rejects_invalid_policy_json_and_version(monkeypatch, tmp_path):
def test_cli_rejects_invalid_policy_json_and_version(monkeypatch, tmp_path, capsys):
from laya import evals_cli

built = _patch_router(monkeypatch)
dataset = _write_dataset(tmp_path, RUN_ROWS)
path = tmp_path / "gates.json"
for content in ("{not json", json.dumps({"version": 2, "rules": [
{"slice": {"language": "zh"}, "metric": "choice_accuracy",
"min_count": 1, "min": 0.5}]})):
"min_count": 1, "min": 0.5}]}),
# Unclosable depth is a document `json` refuses to read, and it says so with a
# `RecursionError` rather than a `ValueError`. It is the caller's file, so `docs/evals.md`
# ("Invalid policy syntax exits 2 before a checkpoint loads") belongs to it too.
"[" * 100000 + "]" * 100000):
path.write_text(content, encoding="utf-8")
assert evals_cli.main(["run", dataset, "--gate-policy", str(path)]) == 2
err = capsys.readouterr().err
assert "Traceback" not in err, "exit 2 with a traceback is the shape this refuses"
assert err.startswith("laya-evals: "), "a usage error, named as one: %r" % err
assert "gate policy" in err, "and named as *which* usage error: %r" % err
# A path that is not there is `OSError`, and `load_policy` names it as a gate policy on purpose:
# `main`'s own `FileNotFoundError` arm would still exit 2, with a message that no longer says
# which flag pointed at the missing file.
missing = tmp_path / "absent-gates.json"
assert evals_cli.main(["run", dataset, "--gate-policy", str(missing)]) == 2
err = capsys.readouterr().err
assert "Traceback" not in err, "exit 2 with a traceback is the shape this refuses: %r" % err
assert err.startswith("laya-evals: ") and "gate policy" in err, (
"the refusal still names the flag it came from: %r" % err)
assert str(missing) in err, "and the path the caller typed: %r" % err
assert not built


Expand Down
Loading