diff --git a/laya/train.py b/laya/train.py index 442ed27c2..a20fa5962 100644 --- a/laya/train.py +++ b/laya/train.py @@ -756,9 +756,10 @@ def calibration_report(records: Sequence[Tuple[int, Any, Any, int]], """Per question type: how many calibration items there were, what was fitted, and what to doubt. `records` are the `calibration_records` the temperatures were fitted on, and `temperature` - the per-type scalars `fit_temperature_map` returned. Each type gets `items`, `temperature`, - `accuracy` and `mean_confidence` (top probability at the fitted temperature) on the - calibration items, and a list of `issues`, empty when nothing looks wrong: + the per-type scalars `fit_temperature_map` returned. Each type gets `items`, `temperature` + and a list of `issues`, empty when nothing looks wrong, and a type that has items also gets + `accuracy` and `mean_confidence` (top probability at the fitted temperature) on them. The + issues: - fewer than `MIN_TYPE_N` items: the temperature was not fitted and stays 1.0; - fewer than `CALIB_WARN_N` items: it was fitted, on little evidence; @@ -766,7 +767,8 @@ def calibration_report(records: Sequence[Tuple[int, Any, Any, int]], - it came back unchanged at 1.0 although it was fitted, which usually means the items gave the fit nothing to correct -- what a model already certain and right on them produces. - A type with no calibration items at all has no issues; it was not in the data. + A type with no calibration items at all has no issues and no metric keys; it was not in the + data, so there is nothing to score it on. """ import numpy as np diff --git a/tests/test_train.py b/tests/test_train.py index e706d0b44..688eef5ee 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -1039,6 +1039,22 @@ def test_clamped_and_healthy_fits(self): p = torch.softmax(torch.tensor([1.0, 0.0, 0.0]) / 1.3, -1) self.assertAlmostEqual(report["score"]["mean_confidence"], p.max().item(), places=6) + def test_metric_keys_follow_the_data(self): + # The docstring lists `accuracy` and `mean_confidence` as per-type entries, but only a + # type that has calibration items gets them; reading one off an absent type is a + # KeyError, so the whole key surface is pinned here rather than one key at a time. + from laya.train import calibration_report + + report = calibration_report(self.records(QTYPES["score"], 60), [1.0, 1.3, 1.0]) + self.assertEqual(sorted(report["score"]), + ["accuracy", "issues", "items", "mean_confidence", "temperature"]) + for absent in ("choice", "noul"): + self.assertEqual(sorted(report[absent]), ["issues", "items", "temperature"], absent) + with self.assertRaises(KeyError): + report[absent]["accuracy"] + with self.assertRaises(KeyError): + report[absent]["mean_confidence"] + def test_finetune_warns_and_records_a_weak_calibration(self): with tempfile.TemporaryDirectory() as d: root = Path(d)