Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions laya/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -756,17 +756,19 @@ def calibration_report(records: Sequence[Tuple[int, Any, Any, int]],
"""Per question type: how many calibration items there were, what was fitted, and what to doubt.

`records` are the `calibration_records` the temperatures were fitted on, and `temperature`
the per-type scalars `fit_temperature_map` returned. Each type gets `items`, `temperature`,
`accuracy` and `mean_confidence` (top probability at the fitted temperature) on the
calibration items, and a list of `issues`, empty when nothing looks wrong:
the per-type scalars `fit_temperature_map` returned. Each type gets `items`, `temperature`
and a list of `issues`, empty when nothing looks wrong, and a type that has items also gets
`accuracy` and `mean_confidence` (top probability at the fitted temperature) on them. The
issues:

- fewer than `MIN_TYPE_N` items: the temperature was not fitted and stays 1.0;
- fewer than `CALIB_WARN_N` items: it was fitted, on little evidence;
- it landed on the `[TEMP_MIN, TEMP_MAX]` clamp;
- it came back unchanged at 1.0 although it was fitted, which usually means the items gave
the fit nothing to correct -- what a model already certain and right on them produces.

A type with no calibration items at all has no issues; it was not in the data.
A type with no calibration items at all has no issues and no metric keys; it was not in the
data, so there is nothing to score it on.
"""
import numpy as np

Expand Down
16 changes: 16 additions & 0 deletions tests/test_train.py
Original file line number Diff line number Diff line change
Expand Up @@ -1039,6 +1039,22 @@ def test_clamped_and_healthy_fits(self):
p = torch.softmax(torch.tensor([1.0, 0.0, 0.0]) / 1.3, -1)
self.assertAlmostEqual(report["score"]["mean_confidence"], p.max().item(), places=6)

def test_metric_keys_follow_the_data(self):
# The docstring lists `accuracy` and `mean_confidence` as per-type entries, but only a
# type that has calibration items gets them; reading one off an absent type is a
# KeyError, so the whole key surface is pinned here rather than one key at a time.
from laya.train import calibration_report

report = calibration_report(self.records(QTYPES["score"], 60), [1.0, 1.3, 1.0])
self.assertEqual(sorted(report["score"]),
["accuracy", "issues", "items", "mean_confidence", "temperature"])
for absent in ("choice", "noul"):
self.assertEqual(sorted(report[absent]), ["issues", "items", "temperature"], absent)
with self.assertRaises(KeyError):
report[absent]["accuracy"]
with self.assertRaises(KeyError):
report[absent]["mean_confidence"]

def test_finetune_warns_and_records_a_weak_calibration(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
Expand Down
Loading