From 15530c111e48e9bd99c36d5f2a92ee6928c64898 Mon Sep 17 00:00:00 2001 From: rukafuu Date: Wed, 23 Sep 2026 12:44:49 -0300 Subject: [PATCH 1/4] feat(eval): add metamorphic option-permutation evaluation --- research/eval/metamorphic.py | 351 +++++++++++++++++++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 research/eval/metamorphic.py diff --git a/research/eval/metamorphic.py b/research/eval/metamorphic.py new file mode 100644 index 000000000..4a8d218ea --- /dev/null +++ b/research/eval/metamorphic.py @@ -0,0 +1,351 @@ +"""Small, reproducible choice-invariance diagnostics (issue #244). + +Run from the checkout: python -m research.eval.metamorphic --help. +Pure transformations/metrics can also be used without a model or gold labels. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +from dataclasses import dataclass +import json +import math +import random +from typing import Any, Callable, Sequence + +from . import laya_eval as harness + + +@dataclass +class MetamorphicCase: + """One canonical choice decision in the existing harness input format. + + Descriptions must stand alone; state/instructions must not refer to labels + or positions. Nonempty descriptions are necessary, not proof of equivalence. + """ + + state: Any + questions: dict + gold_index: int | None = None + + def __post_init__(self): + if len(self.questions) != 1: + raise ValueError("expected exactly one choice question per case") + question = next(iter(self.questions.values())) + criteria = question.get("criteria") + if question.get("type") != "choice" or not isinstance(criteria, dict): + raise ValueError("expected choice criteria as a dictionary") + if len(criteria) < 2: + raise ValueError("at least two options are required") + if any(not isinstance(k, str) or not k.strip() for k in criteria): + raise ValueError("option keys must be nonempty strings") + if any(v is None or (isinstance(v, str) and not v.strip()) + for v in criteria.values()): + raise ValueError("options require standalone nonempty descriptions") + if self.gold_index is not None and ( + type(self.gold_index) is not int or not 0 <= self.gold_index < len(criteria)): + raise ValueError("gold index outside canonical options") + + @property + def option_keys(self): + return list(next(iter(self.questions.values()))["criteria"]) + + def as_pair(self): + return self.state, self.questions + + +@dataclass +class MetamorphicVariant: + """Transformed input plus an explicit bidirectional semantic label mapping.""" + + kind: str + case: MetamorphicCase + canonical_to_transformed: dict[str, str] + + @property + def transformed_to_canonical(self): + return {v: k for k, v in self.canonical_to_transformed.items()} + + def as_record(self): + mapping = self.canonical_to_transformed + inverse = self.transformed_to_canonical + keys = list(mapping) + if len(inverse) != len(mapping) or set(inverse) != set(self.case.option_keys): + raise ValueError("label mapping must be a bijection over transformed options") + return {"kind": self.kind, "case": self.case.as_pair(), + "canonical_to_transformed": dict(mapping), + "transformed_to_canonical": inverse, + "canonical_indices": [keys.index(inverse[k]) for k in self.case.option_keys]} + + +def _transform(case, kind, order, labels): + questions = deepcopy(case.questions) + question = next(iter(questions.values())) + criteria = question["criteria"] + keys = case.option_keys + mapping = dict(zip(keys, labels)) + question["criteria"] = {mapping[keys[i]]: deepcopy(criteria[keys[i]]) for i in order} + gold = None if case.gold_index is None else order.index(case.gold_index) + return MetamorphicVariant(kind, MetamorphicCase(deepcopy(case.state), questions, gold), mapping) + + +def permute_options(case: MetamorphicCase, seed: int = harness.SEED): + """One deterministic nonidentity shuffle; identity falls back to rotation.""" + order = list(range(len(case.option_keys))) + random.Random(seed).shuffle(order) + if order == list(range(len(order))): + order = order[1:] + order[:1] + return _transform(case, "option_order", order, case.option_keys) + + +def _baseline(case): + return _transform(case, "baseline", list(range(len(case.option_keys))), case.option_keys) + + +def make_variants(case, rng: random.Random): + """Adapt the harness tuple format to baseline plus option permutation. + + Label renaming, paraphrases, and other metamorphic transforms are intentionally deferred. + """ + canonical = MetamorphicCase(*case) + return [v.as_record() for v in ( + _baseline(canonical), permute_options(canonical, rng.getrandbits(64)))] + + +def canonicalize(probabilities, canonical_indices): + """Validate a probability vector and restore canonical semantic ordering.""" + values = [float(p) for p in probabilities] + n = len(values) + if not n or any(type(i) is not int for i in canonical_indices) or sorted(canonical_indices) != list(range(n)): + raise ValueError("probabilities and canonical mapping must be a bijection") + if any(not math.isfinite(p) or p < 0 or p > 1 for p in values): + raise ValueError("probabilities must be finite values in [0, 1]") + if not math.isclose(sum(values), 1.0, abs_tol=1e-6, rel_tol=0): + raise ValueError("probabilities must sum to one") + restored = [0.0] * n + for p, index in zip(values, canonical_indices): + restored[index] = p + return restored + + +def distribution_metrics(baseline, variant): + """Drift in canonical space; JS uses natural logs (range 0..ln(2)). + + Ties choose the first canonical option, so reordering tied slots does not + itself count as disagreement. Confidence is the maximum probability; + confidence drift is signed (variant minus baseline). + """ + p = canonicalize(baseline, range(len(baseline))) + q = canonicalize(variant, range(len(baseline))) + middle = [(a + b) / 2 for a, b in zip(p, q)] + js = sum(0.5 * x * math.log(x / m) + for distribution in (p, q) + for x, m in zip(distribution, middle) if x > 0) + drift = [abs(a - b) for a, b in zip(p, q)] + return { + "semantic_agreement": max(range(len(p)), key=p.__getitem__) == + max(range(len(q)), key=q.__getitem__), + "mean_probability_drift": sum(drift) / len(drift), + "max_probability_drift": max(drift), + "js_divergence": max(0.0, js), + "confidence_drift": max(q) - max(p), + } + + +def summarise_pairs(pairs): + """Pair-weighted summary; absent observations are not perfect agreement.""" + if not pairs: + return {"n": 0} + return { + "n": len(pairs), + "semantic_agreement_rate": sum(p["semantic_agreement"] for p in pairs) / len(pairs), + "mean_probability_drift": sum(p["mean_probability_drift"] for p in pairs) / len(pairs), + "max_probability_drift": max(p["max_probability_drift"] for p in pairs), + "mean_js_divergence": sum(p["js_divergence"] for p in pairs) / len(pairs), + "mean_confidence_drift": sum(p["confidence_drift"] for p in pairs) / len(pairs), + "mean_absolute_confidence_drift": sum(abs(p["confidence_drift"]) for p in pairs) / len(pairs), + "worst_confidence_increase_on_disagreement": max( + [0.0] + [p["confidence_drift"] for p in pairs if not p["semantic_agreement"]]), + } + + +def evaluate(cases, score: Callable, gold_indices=None, seed=harness.SEED, + batch_size=16): + """Score cases via ``score(batch) -> probability vectors in presented order``. + + Optional gold indices refer to original option order; entries may be None. + Batches bound inference memory. RNG state and variants do not depend on batch + size. Returned records retain inputs, mappings and full precision vectors. + """ + cases = list(cases) + rng = random.Random(seed) + generated = [make_variants(case, rng) for case in cases] + return _evaluate_generated(cases, generated, score, gold_indices, batch_size) + + +def _evaluate_generated(cases, generated_by_case, score, gold_indices, batch_size): + if batch_size < 1: + raise ValueError("batch_size must be positive") + cases = list(cases) + golds = [None] * len(cases) if gold_indices is None else list(gold_indices) + if len(golds) != len(cases): + raise ValueError("one gold index is required per case") + variants, records = [], [] + for index, (case, gold) in enumerate(zip(cases, golds)): + generated = generated_by_case[index] + keys = list(next(iter(case[1].values()))["criteria"]) + if gold is not None and (type(gold) is not int or not 0 <= gold < len(keys)): + raise ValueError("gold index outside canonical options") + records.append({"index": index, "state": deepcopy(case[0]), + "questions": deepcopy(case[1]), "options": keys, + "gold_index": gold, "variants": []}) + variants.extend((index, v) for v in generated) + for start in range(0, len(variants), batch_size): + batch = variants[start:start + batch_size] + vectors = list(score([v["case"] for _, v in batch])) + if len(vectors) != len(batch): + raise ValueError("scorer returned the wrong number of probability vectors") + for (index, v), raw in zip(batch, vectors): + probs = canonicalize(raw, v["canonical_indices"]) + pred = max(range(len(probs)), key=probs.__getitem__) + records[index]["variants"].append({ + "kind": v["kind"], + "presented_options": list(next(iter(v["case"][1].values()))["criteria"]), + "canonical_indices": v["canonical_indices"], + "canonical_to_transformed": v["canonical_to_transformed"], + "transformed_to_canonical": v["transformed_to_canonical"], + "probabilities": probs, "pred_index": pred, + "pred_label": records[index]["options"][pred], + "confidence": max(probs), + "correct": None if golds[index] is None else pred == golds[index], + }) + return {"report": _report(records), "cases": records} + + +def _report(records): + groups = {"option_order": []} + for record in records: + baseline = record["variants"][0]["probabilities"] + for variant in record["variants"][1:]: + variant["comparison"] = distribution_metrics(baseline, variant["probabilities"]) + groups[variant["kind"]].append(variant["comparison"]) + report = {kind: summarise_pairs(pairs) for kind, pairs in groups.items()} + report["overall"] = summarise_pairs([p for pairs in groups.values() for p in pairs]) + report["quality"] = {} + for kind in ("baseline", "option_order"): + labelled = [v for r in records for v in r["variants"] + if v["kind"] == kind and v["correct"] is not None] + quality = {"n_labelled": len(labelled)} + if labelled: + quality.update(accuracy=sum(v["correct"] for v in labelled) / len(labelled), + ece=harness.ece([v["confidence"] for v in labelled], + [v["correct"] for v in labelled])) + report["quality"][kind] = quality + return report + + +def model_scorer(agent, unclamped=False): + """Reuse the harness's raw logits and per-option-count temperatures.""" + from laya.common import QTYPES + + def score(cases): + return [harness.softmax_t(z, harness.temperature_for( + agent, QTYPES["choice"], len(z), unclamped)) + for z in harness.score_cases(agent, cases)] + return score + + +@dataclass +class MetamorphicResults: + baseline: dict + variants: list[dict] + + +def evaluate_variants(agent, case: MetamorphicCase, variants, *, score=None, + unclamped=False, batch_size=16): + """Evaluate explicit transforms; inject ``score`` for offline stub models. + + With a real agent, call ``agent.model.eval()`` first, as with the harness. + All output vectors are restored to the original case's canonical ordering. + """ + generated = [_baseline(case).as_record()] + for variant in variants: + if variant.kind not in ("option_order",): + raise ValueError("unsupported transformation kind") + if list(variant.canonical_to_transformed) != case.option_keys: + raise ValueError("variant mapping does not match the canonical case") + generated.append(variant.as_record()) + result = _evaluate_generated([case.as_pair()], [generated], + score if score is not None else model_scorer(agent, unclamped), + [case.gold_index], batch_size) + predictions = result["cases"][0]["variants"] + return MetamorphicResults(predictions[0], predictions[1:]) + + +def compare_predictions(*, baseline, variants): + """Report semantic agreement separately from continuous stability metrics. + + No drift threshold or automatic bug classification is applied. + """ + return _report([{"variants": [deepcopy(baseline), *deepcopy(variants)]}]) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", default="convaiinnovations/laya") + parser.add_argument("--subfolder", default=None) + parser.add_argument("--device", default=None) + parser.add_argument("--langs", default="en", help="comma-separated MASSIVE configs, or all") + parser.add_argument("--per-lang", type=int, default=harness.PER_LANG) + parser.add_argument("--n-opts", type=int, default=harness.N_OPTS) + parser.add_argument("--seed", type=int, default=harness.SEED) + parser.add_argument("--batch-size", type=int, default=16) + parser.add_argument("--unclamped", action="store_true") + parser.add_argument("--out", required=True, help="JSON report path") + args = parser.parse_args(argv) + if args.per_lang < 1 or args.n_opts < 2 or args.batch_size < 1: + parser.error("per-lang/batch-size must be positive and n-opts must be at least 2") + langs = (harness.available_languages() if args.langs.strip().lower() == "all" + else list(dict.fromkeys(x.strip() for x in args.langs.split(",") if x.strip()))) + if not langs: + parser.error("no languages selected") + import laya + + agent = laya.load(args.model, device=args.device, subfolder=args.subfolder) + agent.model.eval() + payload: dict[str, Any] = { + "config": {**vars(args), "dataset": harness.DATASET, "split": "test", + "device": str(agent.device), "laya_version": laya.__version__, + "max_len": agent.cfg.get("max_len"), + "head_max_len": agent.cfg.get("head_max_len"), + "temperature": list(agent.temperature_raw if args.unclamped else agent.temperature), + "temperature_by_options": dict(agent.temperature_by_options_raw if args.unclamped + else agent.temperature_by_options), + "permutations_per_case": 1, "js_log_base": "e"}, + "report": {}, "cases": [], + } + failed = False + for lang in langs: + try: + rows = harness.load_language(lang) + cases, gold, _ = harness.build_suite( + rows, sorted({r["label_text"] for r in rows}), args.per_lang, args.n_opts, args.seed) + if not cases: + raise ValueError("dataset returned no cases") + result = evaluate(cases, model_scorer(agent, args.unclamped), gold, + args.seed, args.batch_size) + payload["report"][lang] = result["report"] + payload["cases"].extend({"lang": lang, **r} for r in result["cases"]) + print(lang, json.dumps(result["report"]), flush=True) + except Exception as exc: + failed = True + payload["report"][lang] = {"error": str(exc)} + print("%s FAILED: %s" % (lang, exc), flush=True) + with open(args.out, "w", encoding="utf-8") as stream: + json.dump(payload, stream, ensure_ascii=False, indent=2, allow_nan=False) + stream.write("\n") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 5b4c48994c91db47cc6040d9de1a93e55425a2df Mon Sep 17 00:00:00 2001 From: rukafuu Date: Wed, 23 Sep 2026 12:45:06 -0300 Subject: [PATCH 2/4] test(eval): cover option permutation invariance offline --- research/eval/test_metamorphic.py | 195 ++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 research/eval/test_metamorphic.py diff --git a/research/eval/test_metamorphic.py b/research/eval/test_metamorphic.py new file mode 100644 index 000000000..f34e408d4 --- /dev/null +++ b/research/eval/test_metamorphic.py @@ -0,0 +1,195 @@ +"""Offline tests: python -m unittest research.eval.test_metamorphic -v.""" +from contextlib import redirect_stdout +from copy import deepcopy +from io import StringIO +import json +import math +from pathlib import Path +import random +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from research.eval import metamorphic as m + +CASE = ({"utterance": "I was charged twice"}, {"intent": { + "type": "choice", "instructions": "Choose the request category.", + "criteria": {"billing": "payment issues", "technical": "software help", "sales": "new purchases"}, +}}) + + +def semantic_scorer(cases): + weights = {"payment issues": 0.7, "software help": 0.2, "new purchases": 0.1} + return [[weights[v] for v in q["intent"]["criteria"].values()] for _, q in cases] + + +class TransformTests(unittest.TestCase): + def test_mapping_and_unchanged_inputs(self): + original = deepcopy(CASE) + baseline, order = m.make_variants(CASE, random.Random(13)) + self.assertEqual(CASE, original) + self.assertEqual(baseline["case"], CASE) + self.assertNotEqual(order["canonical_indices"], [0, 1, 2]) + for variant in (baseline, order): + criteria = variant["case"][1]["intent"]["criteria"] + descriptions = list(CASE[1]["intent"]["criteria"].values()) + self.assertEqual(list(criteria.values()), + [descriptions[i] for i in variant["canonical_indices"]]) + order["case"][0]["utterance"] = "changed" + self.assertEqual(CASE, original) + + def test_identity_shuffle_fallback(self): + with patch("random.Random.shuffle", return_value=None): + variant = m.permute_options(m.MetamorphicCase(*CASE), seed=42) + self.assertEqual(variant.as_record()["canonical_indices"], [1, 2, 0]) + + def test_explicit_api_mapping_and_gold(self): + case = m.MetamorphicCase(*CASE, gold_index=0) + permuted = m.permute_options(case, seed=42) + self.assertEqual(permuted, m.permute_options(case, seed=42)) + self.assertEqual(permuted.canonical_to_transformed, + {"billing": "billing", "technical": "technical", "sales": "sales"}) + self.assertEqual(permuted.transformed_to_canonical["billing"], "billing") + self.assertEqual(permuted.case.option_keys[permuted.case.gold_index], "billing") + result = m.evaluate_variants(None, case, [permuted], score=semantic_scorer) + report = m.compare_predictions(baseline=result.baseline, variants=result.variants) + self.assertEqual(report["overall"]["semantic_agreement_rate"], 1) + self.assertEqual(report["overall"]["max_probability_drift"], 0) + + def test_many_options_and_nested_descriptions(self): + case = deepcopy(CASE) + case[1]["intent"]["criteria"] = {str(i): {"description": [str(i)]} for i in range(28)} + order = m.make_variants(case, random.Random(0))[1] + criteria = order["case"][1]["intent"]["criteria"] + self.assertEqual(len(criteria), 28) + + def test_invalid_cases(self): + for criteria in ({}, {"a": "one"}, {"a": None, "b": "two"}, + {"a": " ", "b": "two"}, {1: "one", "b": "two"}): + case = deepcopy(CASE) + case[1]["intent"]["criteria"] = criteria + with self.subTest(criteria=criteria), self.assertRaises(ValueError): + m.make_variants(case, random.Random(0)) + for questions in ({}, {"a": CASE[1]["intent"], "b": CASE[1]["intent"]}, + {"a": {"type": "noul", "criteria": {"false": "no", "true": "yes"}}}): + with self.assertRaises(ValueError): + m.make_variants(({}, questions), random.Random(0)) + + +class MetricTests(unittest.TestCase): + def test_inverse_mapping(self): + self.assertEqual(m.canonicalize([0.1, 0.7, 0.2], [2, 0, 1]), [0.7, 0.2, 0.1]) + + def test_invalid_distributions(self): + for vector, mapping in (([], []), ([0.2, 0.8], [0, 0]), + ([0.2, 0.8], [0]), ([0.2, 0.8], [0, 2]), + ([float("nan"), 0.5], [0, 1]), + ([float("inf"), 0], [0, 1]), + ([-0.1, 1.1], [0, 1]), ([0.2, 0.2], [0, 1])): + with self.subTest(vector=vector), self.assertRaises(ValueError): + m.canonicalize(vector, mapping) + + def test_known_divergence_and_drift(self): + result = m.distribution_metrics([1, 0], [0, 1]) + self.assertFalse(result["semantic_agreement"]) + self.assertAlmostEqual(result["js_divergence"], math.log(2)) + self.assertEqual(result["mean_probability_drift"], 1) + self.assertEqual(result["confidence_drift"], 0) + identical = m.distribution_metrics([0.5, 0.5, 0], [0.5, 0.5, 0]) + self.assertTrue(identical["semantic_agreement"]) + self.assertEqual(identical["js_divergence"], 0) + + def test_agreement_is_distinct_from_stability(self): + same_winner = m.distribution_metrics([0.91, 0.06, 0.03], [0.88, 0.08, 0.04]) + self.assertTrue(same_winner["semantic_agreement"]) + self.assertGreater(same_winner["mean_probability_drift"], 0) + self.assertGreater(same_winner["js_divergence"], 0) + changed_winner = m.distribution_metrics([0.91, 0.06, 0.03], [0.08, 0.86, 0.06]) + self.assertFalse(changed_winner["semantic_agreement"]) + self.assertLess(changed_winner["confidence_drift"], 0) + + def test_summary_and_confidence_increase(self): + pairs = [m.distribution_metrics([0.6, 0.4], [0.1, 0.9]), m.distribution_metrics([0.6, 0.4], [0.8, 0.2])] + summary = m.summarise_pairs(pairs) + self.assertEqual(summary["semantic_agreement_rate"], 0.5) + self.assertAlmostEqual(summary["mean_probability_drift"], 0.35) + self.assertAlmostEqual(summary["max_probability_drift"], 0.5) + self.assertAlmostEqual(summary["mean_confidence_drift"], 0.25) + self.assertAlmostEqual(summary["worst_confidence_increase_on_disagreement"], 0.3) + self.assertEqual(m.summarise_pairs([]), {"n": 0}) + decreasing = m.summarise_pairs([m.distribution_metrics([0.9, 0.1], [0.4, 0.6])]) + self.assertEqual(decreasing["worst_confidence_increase_on_disagreement"], 0) + + +class EvaluationTests(unittest.TestCase): + def test_invariance_batching_seed_gold_and_json(self): + outputs = [] + for batch_size in (1, 2, 16): + sizes = [] + def score(cases): + sizes.append(len(cases)) + return semantic_scorer(cases) + result = m.evaluate([CASE, CASE], score, [0, None], batch_size=batch_size) + self.assertLessEqual(max(sizes), batch_size) + self.assertEqual(result["report"]["overall"]["semantic_agreement_rate"], 1) + self.assertEqual(result["report"]["overall"]["max_probability_drift"], 0) + self.assertEqual(result["report"]["quality"]["baseline"]["accuracy"], 1) + self.assertEqual(result["report"]["quality"]["baseline"]["n_labelled"], 1) + self.assertAlmostEqual(result["report"]["quality"]["baseline"]["ece"], 0.3) + outputs.append(result) + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[0], outputs[2]) + self.assertEqual(json.loads(json.dumps(outputs[0], allow_nan=False)), outputs[0]) + + def test_position_bias_detected(self): + report = m.evaluate([CASE], lambda cases: [[1, 0, 0] for _ in cases])["report"] + self.assertEqual(report["option_order"]["semantic_agreement_rate"], 0) + self.assertEqual(report["quality"]["baseline"], {"n_labelled": 0}) + + def test_ties_use_canonical_order(self): + result = m.evaluate([CASE], lambda cases: [[1/3] * 3 for _ in cases]) + self.assertEqual(result["report"]["overall"]["semantic_agreement_rate"], 1) + + def test_empty_bad_scorers_and_gold(self): + self.assertEqual(m.evaluate([], lambda _: self.fail("should not score"))["report"]["overall"], {"n": 0}) + for score in (lambda cases: [], lambda cases: [[1, 0] for _ in cases]): + with self.assertRaises(ValueError): + m.evaluate([CASE], score) + for gold in ([], [3], [-1], [True], [0.5]): + with self.assertRaises(ValueError): + m.evaluate([CASE], semantic_scorer, gold) + with self.assertRaises(ValueError): + m.evaluate([CASE], semantic_scorer, batch_size=0) + + def test_model_adapter_temperature(self): + class Agent: + temperature = [1.0] * 3 + temperature_raw = [0.5] * 3 + temperature_by_options = {} + temperature_by_options_raw = {} + with patch.object(m.harness, "score_cases", return_value=[[0, math.log(3)]]): + self.assertAlmostEqual(m.model_scorer(Agent())([CASE])[0][1], 0.75) + self.assertAlmostEqual(m.model_scorer(Agent(), True)([CASE])[0][1], 0.9) + + def test_cli_partial_failure_writes_report_and_fails(self): + agent = MagicMock() + agent.device = "cpu" + agent.cfg = {"max_len": 512, "head_max_len": 192} + agent.temperature = [1.0] * 3 + agent.temperature_by_options = {} + rows = [{"text": "x", "label_text": "a"}, {"text": "y", "label_text": "b"}] + with tempfile.TemporaryDirectory() as folder: + out = Path(folder) / "report.json" + with redirect_stdout(StringIO()), patch("laya.load", return_value=agent), \ + patch.object(m.harness, "load_language", side_effect=[rows, RuntimeError("offline")]), \ + patch.object(m, "model_scorer", return_value=lambda cases: [[0.6, 0.4] for _ in cases]): + code = m.main(["--langs", "en,de", "--n-opts", "2", "--out", str(out)]) + payload = json.loads(out.read_text(encoding="utf-8")) + self.assertEqual(code, 1) + self.assertEqual(payload["report"]["de"], {"error": "offline"}) + self.assertEqual(len(payload["cases"]), 2) + self.assertEqual(payload["config"]["temperature"], [1.0] * 3) + + +if __name__ == "__main__": + unittest.main() From c18d8e5bde8928343e8220f91635dcbd3577fffe Mon Sep 17 00:00:00 2001 From: rukafuu Date: Wed, 23 Sep 2026 12:46:31 -0300 Subject: [PATCH 3/4] docs(eval): document option-order robustness evaluation --- research/eval/README.md | 120 ++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 48 deletions(-) diff --git a/research/eval/README.md b/research/eval/README.md index 6470bbda9..ac5a779bc 100644 --- a/research/eval/README.md +++ b/research/eval/README.md @@ -137,24 +137,8 @@ having an empty `temperature_by_options`, so the clamp cannot explain it. Ruled out: the option sets (identical digest to the english run), the weights (bundled and standalone multilingual are byte-identical, all 170 tensors `torch.equal`), the dataset (revision `940fd47a`, last modified 2026-02-24), and -`build_sequence` (unchanged since `v0.2.0`). Also ruled out, on re-measurement: - -* **the shipped `head_max_len`**, which matters here because this checkpoint ships - `256` and english ships `192`. The harness reads it from the checkpoint's own - config and the run's `config` block records `head_max_len: 256, max_len: 1024`, so - the multilingual numbers above were not taken at english's budget. Re-running with - the value read from config gives the same `0.4008`, and `6/51` again. -* **which of the two multilingual copies was measured.** The bundled `multilingual/` - subfolder and the standalone `convaiinnovations/laya-multilingual` repo were each - run end to end over all 51 languages and both give `macro_accuracy 0.4008`, - `macro_ece 0.3911`, `6/51`. -* **a checkpoint change since the committed sweep.** `multilingual/model.safetensors` - is `643835514` bytes at `sha256 b99c8bea…` and `multilingual/rl_agent_config.json` - is `472` bytes at `sha256 00e35f88…` at every revision from the sweep's timestamp to - today; the Hub commits in that window are model-card `docs:`/`assets:` only. - -It is in the multilingual inference path between `laya 0.2.0` and `0.3.6` and is -**not** reconciled. Flagged rather than hidden. +`build_sequence` (unchanged since `v0.2.0`). It is in the multilingual inference path +between `laya 0.2.0` and `0.3.6` and is **not** reconciled. Flagged rather than hidden. Related: **`head_max_len` is load-bearing for accuracy**, not just for option truncation. The english checkpoint at its shipped `head_max_len=192` scores 0.82; @@ -166,15 +150,13 @@ forcing 256 or 512 drops it to 0.79. checkpoint, no network: ```bash -python research/eval/test_laya_eval.py # 64 passed, 0 failed +python research/eval/test_laya_eval.py # 47 passed, 0 failed ``` It pins the upstream constants (seed 13, 20 options, the exact instruction string), the determinism of the sampler, that a fresh RNG per language is used, and the metric arithmetic, including the `confidence == 0.0` bin boundary that this harness -shares with `laya.common.ece_score`, `research/scripts/bench_local.py` and -`research/scripts/build_benchmark_nb.py`. That boundary is asserted against all four, -not just against this harness's own arithmetic. +shares with `bench_local.py` and `laya.common.ece_score`. ## Limits @@ -185,32 +167,74 @@ not just against this harness's own arithmetic. * The English checkpoint collapses on non-Latin scripts (see `BENCHMARKS.md`), so a low score in one language is not by itself evidence of a misroute — check `laya.lang.analyse` for the script before concluding which checkpoint was used. -* The `confidence == 0.0` bin boundary is the one - [#39](https://github.com/NandhaKishorM/laya/pull/39) settled: the first bin is closed - at the bottom, so `0.0` is counted. This harness used `conf > lo` for every bin until - the divergence was found, which made it the only one of the four implementations that - binned differently. It now matches `laya.common.ece_score`, - `research/scripts/bench_local.py` and `research/scripts/build_benchmark_nb.py`, and - `test_laya_eval.py` asserts that agreement. +* `confidence == 0.0` is not binned, matching `laya.common.ece_score` and + `bench_local.py`. If [#39](https://github.com/NandhaKishorM/laya/pull/39) lands + with a different boundary, this should follow it. -### The temperature clamp, measured both ways +## Metamorphic robustness (experimental) -`research/results/cpu_51_language_sweep_clamped.json` carries the same re-run twice, once per -regime, against the committed columns. Macro accuracy reproduces the committed file exactly -and macro ECE is the only macro figure that moves: +`metamorphic.py` adds the initial scope of +[#244](https://github.com/NandhaKishorM/laya/issues/244): **choice option order robustness**, without changing model/runtime behavior. Label renaming, neutral labels, paraphrases, structured-state permutations, `score` and `noul` perturbations are intentionally deferred. Run from the repository root after installing Laya and `datasets`: -| | committed | `--unclamped` | default | -|---|---|---|---| -| `macro_accuracy` | 0.2269 | **0.2269** | 0.2269 | -| `macro_ece` | 0.7331 | **0.7331** | 0.5709 | -| `macro_f1` | 0.2053 | **0.2053** | 0.2053 | - -Per language, the unclamped run agrees with the committed file on `accuracy` and `macro_f1` -in **51/51**, on `ece` in **48/51** and on `mean_confidence` in **49/51**. The handful that -differ do so by `0.0001`, the last stored digit: the committed run used torch 2.8.0 and this -one 2.14.0. The clamped run differs from the committed file on `ece` and `mean_confidence` in -**51/51**, every one of them lower, because it is the only column the clamp can move. - -`accuracy`, `macro_f1` and `n` are identical in all three columns by construction: scaling -logits by any positive temperature does not change the argmax. That is why a re-run can settle -the calibration question without reopening the accuracy numbers. +```bash +python -m research.eval.metamorphic --model convaiinnovations/laya \ + --langs en --per-lang 100 --n-opts 20 --batch-size 16 --out robustness.json +python -m research.eval.metamorphic --model convaiinnovations/laya \ + --subfolder multilingual --langs en --out multilingual-robustness.json +python -m unittest research.eval.test_metamorphic -v +``` + +Each MASSIVE case uses the existing harness's sampler and produces two inputs: + +1. The unchanged baseline. +2. One seeded shuffle of option order; if the shuffle is the identity, a one-slot + rotation is used. This is a bounded diagnostic, not exhaustive permutation testing + or a uniform draw over all nonidentity permutations. + +Descriptions, instructions and state are otherwise unchanged. Every result is mapped back to the original semantic option order **before** predictions and metrics are computed. Exact ties choose the first canonical option. The RNG starts fresh per language; `--seed` controls both sampling and transformations. `--batch-size` bounds the number of forward-pass inputs and does not alter the generated variants. Model inference may still have small floating-point differences across devices and batch sizes. + +The JSON contains `config`, per-language `report`, and full `cases`. Each case saves its original input, canonical keys and optional gold index; each variant saves its presented keys, explicit `canonical_to_transformed` and `transformed_to_canonical` label mappings, slot-to-canonical indices, complete **canonical-order** probability vector, prediction, confidence, correctness (or `null`), and comparison to baseline. Probabilities are not rounded. The config records model/subfolder, temperature mode and values, truncation settings, dataset, seed and batch size. For reproducible checkpoint comparisons, use a pinned local snapshot and retain the environment versions alongside the report. `--unclamped` has the same meaning as in `laya_eval`. If any language fails, its error is saved and the command exits nonzero while retaining successful languages. + +Metrics are grouped under `option_order` and `overall`: + +| Metric | Definition | +|---|---| +| `semantic_agreement_rate` | Fraction of baseline/variant pairs with the same canonical argmax | +| `mean_probability_drift` | Mean absolute probability change across options, then pairs | +| `max_probability_drift` | Largest absolute change of any option across all pairs | +| `mean_js_divergence` | Mean Jensen-Shannon divergence using natural logs, in `[0, ln(2)]` | +| `mean_confidence_drift` | Mean signed change of maximum probability, variant minus baseline | +| `mean_absolute_confidence_drift` | Mean magnitude of that confidence change | +| `worst_confidence_increase_on_disagreement` | Largest positive confidence change among changed decisions, or zero if none | + +`overall` is pair-weighted, not a fraction of cases where *all* variants agree. `quality` separately reports accuracy and the existing harness's 15-bin ECE for baseline and each transformation on labelled cases only. Empty groups contain `n: 0`; unlabelled quality groups contain `n_labelled: 0` without inventing an accuracy or ECE. Robustness agreement is not a correctness measure: consistently wrong predictions can be perfectly invariant. + +For another corpus, the Python API accepts `(state, questions)` cases in the same shape as the harness, and a callback returning probability vectors in presented option order: + +```python +from research.eval.metamorphic import evaluate, model_scorer +agent.model.eval() +result = evaluate(cases, model_scorer(agent), gold_indices=None, seed=13) +``` + +The first version intentionally accepts only **one choice question per case**, with at least two options and nonempty descriptions. The caller must ensure descriptions are self-contained and neither state nor instructions refers to option keys or positions. Label renaming and other metamorphic transforms are intentionally deferred as proposed in the issue. + +For an explicit single-case experiment, the same implementation exposes: + +```python +from research.eval.metamorphic import ( + MetamorphicCase, permute_options, + evaluate_variants, compare_predictions, +) + +case = MetamorphicCase(state, questions, gold_index=None) +variants = [permute_options(case, seed=42)] +agent.model.eval() +results = evaluate_variants(agent, case, variants) +report = compare_predictions(baseline=results.baseline, variants=results.variants) +``` + +For offline tests, pass `agent=None, score=fake_scorer` to `evaluate_variants`. The scorer takes a batch of harness `(state, questions)` inputs and returns one probability vector per input. The public transformations return independent copies and explicit mappings in both directions, including identity label mappings for order-only transformations. + +**Semantic agreement and distribution stability are different properties.** +A shift from `[0.91, 0.06, 0.03]` to `[0.88, 0.08, 0.04]` preserves the decision while showing nonzero drift. Switching the winner is reported as disagreement, regardless of whether confidence rises or falls. No metric here automatically classifies either observation as a bug; acceptable variation depends on the use case, and the report deliberately defines no universal pass/fail threshold. \ No newline at end of file From 4b09a292fdb1e4f19ef13f504d40cfc80463ecea Mon Sep 17 00:00:00 2001 From: rukafuu Date: Wed, 23 Sep 2026 12:58:10 -0300 Subject: [PATCH 4/4] fix(eval): preserve upstream docs and relax permutation criteria --- research/eval/README.md | 67 ++++++++++++++++++++++++++----- research/eval/metamorphic.py | 9 +---- research/eval/test_metamorphic.py | 10 ++++- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/research/eval/README.md b/research/eval/README.md index ac5a779bc..af18aa300 100644 --- a/research/eval/README.md +++ b/research/eval/README.md @@ -137,8 +137,24 @@ having an empty `temperature_by_options`, so the clamp cannot explain it. Ruled out: the option sets (identical digest to the english run), the weights (bundled and standalone multilingual are byte-identical, all 170 tensors `torch.equal`), the dataset (revision `940fd47a`, last modified 2026-02-24), and -`build_sequence` (unchanged since `v0.2.0`). It is in the multilingual inference path -between `laya 0.2.0` and `0.3.6` and is **not** reconciled. Flagged rather than hidden. +`build_sequence` (unchanged since `v0.2.0`). Also ruled out, on re-measurement: + +* **the shipped `head_max_len`**, which matters here because this checkpoint ships + `256` and english ships `192`. The harness reads it from the checkpoint's own + config and the run's `config` block records `head_max_len: 256, max_len: 1024`, so + the multilingual numbers above were not taken at english's budget. Re-running with + the value read from config gives the same `0.4008`, and `6/51` again. +* **which of the two multilingual copies was measured.** The bundled `multilingual/` + subfolder and the standalone `convaiinnovations/laya-multilingual` repo were each + run end to end over all 51 languages and both give `macro_accuracy 0.4008`, + `macro_ece 0.3911`, `6/51`. +* **a checkpoint change since the committed sweep.** `multilingual/model.safetensors` + is `643835514` bytes at `sha256 b99c8bea…` and `multilingual/rl_agent_config.json` + is `472` bytes at `sha256 00e35f88…` at every revision from the sweep's timestamp to + today; the Hub commits in that window are model-card `docs:`/`assets:` only. + +It is in the multilingual inference path between `laya 0.2.0` and `0.3.6` and is +**not** reconciled. Flagged rather than hidden. Related: **`head_max_len` is load-bearing for accuracy**, not just for option truncation. The english checkpoint at its shipped `head_max_len=192` scores 0.82; @@ -150,13 +166,15 @@ forcing 256 or 512 drops it to 0.79. checkpoint, no network: ```bash -python research/eval/test_laya_eval.py # 47 passed, 0 failed +python research/eval/test_laya_eval.py # 64 passed, 0 failed ``` It pins the upstream constants (seed 13, 20 options, the exact instruction string), the determinism of the sampler, that a fresh RNG per language is used, and the metric arithmetic, including the `confidence == 0.0` bin boundary that this harness -shares with `bench_local.py` and `laya.common.ece_score`. +shares with `laya.common.ece_score`, `research/scripts/bench_local.py` and +`research/scripts/build_benchmark_nb.py`. That boundary is asserted against all four, +not just against this harness's own arithmetic. ## Limits @@ -167,11 +185,38 @@ shares with `bench_local.py` and `laya.common.ece_score`. * The English checkpoint collapses on non-Latin scripts (see `BENCHMARKS.md`), so a low score in one language is not by itself evidence of a misroute — check `laya.lang.analyse` for the script before concluding which checkpoint was used. -* `confidence == 0.0` is not binned, matching `laya.common.ece_score` and - `bench_local.py`. If [#39](https://github.com/NandhaKishorM/laya/pull/39) lands - with a different boundary, this should follow it. +* The `confidence == 0.0` bin boundary is the one + [#39](https://github.com/NandhaKishorM/laya/pull/39) settled: the first bin is closed + at the bottom, so `0.0` is counted. This harness used `conf > lo` for every bin until + the divergence was found, which made it the only one of the four implementations that + binned differently. It now matches `laya.common.ece_score`, + `research/scripts/bench_local.py` and `research/scripts/build_benchmark_nb.py`, and + `test_laya_eval.py` asserts that agreement. -## Metamorphic robustness (experimental) +### The temperature clamp, measured both ways + +`research/results/cpu_51_language_sweep_clamped.json` carries the same re-run twice, once per +regime, against the committed columns. Macro accuracy reproduces the committed file exactly +and macro ECE is the only macro figure that moves: + +| | committed | `--unclamped` | default | +|---|---|---|---| +| `macro_accuracy` | 0.2269 | **0.2269** | 0.2269 | +| `macro_ece` | 0.7331 | **0.7331** | 0.5709 | +| `macro_f1` | 0.2053 | **0.2053** | 0.2053 | + +Per language, the unclamped run agrees with the committed file on `accuracy` and `macro_f1` +in **51/51**, on `ece` in **48/51** and on `mean_confidence` in **49/51**. The handful that +differ do so by `0.0001`, the last stored digit: the committed run used torch 2.8.0 and this +one 2.14.0. The clamped run differs from the committed file on `ece` and `mean_confidence` in +**51/51**, every one of them lower, because it is the only column the clamp can move. + +`accuracy`, `macro_f1` and `n` are identical in all three columns by construction: scaling +logits by any positive temperature does not change the argmax. That is why a re-run can settle +the calibration question without reopening the accuracy numbers. + + +## Metamorphic option-order robustness (experimental) `metamorphic.py` adds the initial scope of [#244](https://github.com/NandhaKishorM/laya/issues/244): **choice option order robustness**, without changing model/runtime behavior. Label renaming, neutral labels, paraphrases, structured-state permutations, `score` and `noul` perturbations are intentionally deferred. Run from the repository root after installing Laya and `datasets`: @@ -191,7 +236,7 @@ Each MASSIVE case uses the existing harness's sampler and produces two inputs: rotation is used. This is a bounded diagnostic, not exhaustive permutation testing or a uniform draw over all nonidentity permutations. -Descriptions, instructions and state are otherwise unchanged. Every result is mapped back to the original semantic option order **before** predictions and metrics are computed. Exact ties choose the first canonical option. The RNG starts fresh per language; `--seed` controls both sampling and transformations. `--batch-size` bounds the number of forward-pass inputs and does not alter the generated variants. Model inference may still have small floating-point differences across devices and batch sizes. +Instructions and state are otherwise unchanged. Option key/value pairs are moved together during permutation. Every result is mapped back to the original semantic option order **before** predictions and metrics are computed. Exact ties choose the first canonical option. The RNG starts fresh per language; `--seed` controls both sampling and transformations. `--batch-size` bounds the number of forward-pass inputs and does not alter the generated variants. Model inference may still have small floating-point differences across devices and batch sizes. The JSON contains `config`, per-language `report`, and full `cases`. Each case saves its original input, canonical keys and optional gold index; each variant saves its presented keys, explicit `canonical_to_transformed` and `transformed_to_canonical` label mappings, slot-to-canonical indices, complete **canonical-order** probability vector, prediction, confidence, correctness (or `null`), and comparison to baseline. Probabilities are not rounded. The config records model/subfolder, temperature mode and values, truncation settings, dataset, seed and batch size. For reproducible checkpoint comparisons, use a pinned local snapshot and retain the environment versions alongside the report. `--unclamped` has the same meaning as in `laya_eval`. If any language fails, its error is saved and the command exits nonzero while retaining successful languages. @@ -217,7 +262,7 @@ agent.model.eval() result = evaluate(cases, model_scorer(agent), gold_indices=None, seed=13) ``` -The first version intentionally accepts only **one choice question per case**, with at least two options and nonempty descriptions. The caller must ensure descriptions are self-contained and neither state nor instructions refers to option keys or positions. Label renaming and other metamorphic transforms are intentionally deferred as proposed in the issue. +The first version intentionally accepts only **one choice question per case**, with at least two options. Label renaming and other metamorphic transforms are intentionally deferred as proposed in the issue. For an explicit single-case experiment, the same implementation exposes: @@ -237,4 +282,4 @@ report = compare_predictions(baseline=results.baseline, variants=results.variant For offline tests, pass `agent=None, score=fake_scorer` to `evaluate_variants`. The scorer takes a batch of harness `(state, questions)` inputs and returns one probability vector per input. The public transformations return independent copies and explicit mappings in both directions, including identity label mappings for order-only transformations. **Semantic agreement and distribution stability are different properties.** -A shift from `[0.91, 0.06, 0.03]` to `[0.88, 0.08, 0.04]` preserves the decision while showing nonzero drift. Switching the winner is reported as disagreement, regardless of whether confidence rises or falls. No metric here automatically classifies either observation as a bug; acceptable variation depends on the use case, and the report deliberately defines no universal pass/fail threshold. \ No newline at end of file +A shift from `[0.91, 0.06, 0.03]` to `[0.88, 0.08, 0.04]` preserves the decision while showing nonzero drift. Switching the winner is reported as disagreement, regardless of whether confidence rises or falls. No metric here automatically classifies either observation as a bug; acceptable variation depends on the use case, and the report deliberately defines no universal pass/fail threshold. diff --git a/research/eval/metamorphic.py b/research/eval/metamorphic.py index 4a8d218ea..097ee1c54 100644 --- a/research/eval/metamorphic.py +++ b/research/eval/metamorphic.py @@ -18,11 +18,7 @@ @dataclass class MetamorphicCase: - """One canonical choice decision in the existing harness input format. - - Descriptions must stand alone; state/instructions must not refer to labels - or positions. Nonempty descriptions are necessary, not proof of equivalence. - """ + """One canonical choice decision in the existing harness input format.""" state: Any questions: dict @@ -39,9 +35,6 @@ def __post_init__(self): raise ValueError("at least two options are required") if any(not isinstance(k, str) or not k.strip() for k in criteria): raise ValueError("option keys must be nonempty strings") - if any(v is None or (isinstance(v, str) and not v.strip()) - for v in criteria.values()): - raise ValueError("options require standalone nonempty descriptions") if self.gold_index is not None and ( type(self.gold_index) is not int or not 0 <= self.gold_index < len(criteria)): raise ValueError("gold index outside canonical options") diff --git a/research/eval/test_metamorphic.py b/research/eval/test_metamorphic.py index f34e408d4..3ac16679a 100644 --- a/research/eval/test_metamorphic.py +++ b/research/eval/test_metamorphic.py @@ -63,9 +63,15 @@ def test_many_options_and_nested_descriptions(self): criteria = order["case"][1]["intent"]["criteria"] self.assertEqual(len(criteria), 28) + def test_none_or_blank_descriptions_accepted(self): + for criteria in ({"a": None, "b": "two"}, {"a": " ", "b": ""}, {"a": None, "b": None}): + case = deepcopy(CASE) + case[1]["intent"]["criteria"] = criteria + variants = m.make_variants(case, random.Random(0)) + self.assertEqual(len(variants), 2) + def test_invalid_cases(self): - for criteria in ({}, {"a": "one"}, {"a": None, "b": "two"}, - {"a": " ", "b": "two"}, {1: "one", "b": "two"}): + for criteria in ({}, {"a": "one"}, {1: "one", "b": "two"}, {" ": "one", "b": "two"}, {"": "one", "b": "two"}): case = deepcopy(CASE) case[1]["intent"]["criteria"] = criteria with self.subTest(criteria=criteria), self.assertRaises(ValueError):