diff --git a/README.md b/README.md index 16744f965..531cdad05 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,11 @@ print("Churn Risk :", answers["churn_risk"]["noul"]) # -> 0.892 (89.2% pro ## Automated Confidence Gating +For a measured error and coverage requirement, use the opt-in +[threshold selection and independent testing utilities](docs/risk_policy.md). +They account for sample uncertainty and multiple threshold comparisons, and require +separate evidence for every declared language. + Because Laya's probabilities are trained with strictly proper scoring rules (RLCD), confidence scores are statistically meaningful: ```python diff --git a/docs/risk_policy.md b/docs/risk_policy.md new file mode 100644 index 000000000..3f9c13e75 --- /dev/null +++ b/docs/risk_policy.md @@ -0,0 +1,113 @@ +# Selecting and testing review thresholds + +`laya.risk` selects categorical decision thresholds and evaluates them on a +separate final test. It is an opt-in, offline utility. It leaves `Agent.predict` +unchanged and does not itself authorize deployment or load a model. + +The score used here is the largest class probability. The SDK's `confidence` +field is based on normalized entropy; do not substitute it for the probability +vector. High scores alone are insufficient evidence of a low error rate. + +## Input contract + +Supply independent labelled predictions for one fixed categorical question, with +one record per connected example family and language: + +```json +{ + "id": "ticket:42:en", + "group_id": "conversation:42", + "language": "en", + "probabilities": [0.91, 0.09], + "target": 0 +} +``` + +Use the full, unrounded probability vector in the fixed option order and an +integer gold option index. Vectors need at least two finite, nonnegative values +and must sum to one within `1e-6` numerical tolerance. Ties choose the first +option, matching argmax. Repeated IDs or repeated groups within a language fail +validation. Groups can appear in several languages, but must remain entirely +within one partition. Group IDs and example IDs must be globally unique across +data sources; keep translation, conversation and augmentation families together. + +Select representatives before inspecting predictions. Correct grouping and a +representative sample are prerequisites; declaring different IDs cannot make +dependent examples independent. If using the adaptation dataset contract, use +its connected-group IDs rather than the raw, unmerged source family IDs. + +## Selection and independent evaluation + +Freeze the model, question and option definitions, calibration, required +languages, error budget, coverage target and threshold grid before examining +selection outcomes. The defaults are a 5% error budget, at least 50% coverage and +a total failure probability `alpha=0.05`. The grid must be fixed independently +of the selection-set outcomes; do not tune it after seeing which trials pass. + +```python +from laya.risk import evaluate_risk_policy, select_risk_policy + +# selection_samples and final_samples are independently prepared predictions. +selection = select_risk_policy( + selection_samples, + required_languages=["en", "de"], + thresholds=[0.0, 0.5, 0.7, 0.8, 0.9, 0.95, 0.99], + max_error=0.05, + min_coverage=0.5, + alpha=0.05, +) +if selection["passed"]: + final_report = evaluate_risk_policy(final_samples, selection) + print(final_report["passed"]) +``` + +An example is selected when its largest probability is at least the threshold. +For each language and candidate, the report contains: + +| Field | Meaning | +| --- | --- | +| `samples`, `accepted`, `errors` | Independent representatives, selected decisions and incorrect selected decisions | +| `coverage`, `error_rate` | Observed selected fraction and selected error fraction; error is `null` if none are selected | +| `error_upper` | One-sided exact binomial upper confidence bound on selected error | +| `coverage_lower` | One-sided exact binomial lower confidence bound on coverage | +| `qualified` | Both bounds meet their respective predeclared budgets | + +Bounds use the [Clopper-Pearson method](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats._result_classes.BinomTestResult.proportion_ci.html), +implemented by binomial CDF inversion without adding SciPy as a dependency. +Selection allocates `alpha / (2 * languages * thresholds)` to each bound, so +Bonferroni accounts for choosing among multiple candidates and reporting multiple +languages and metrics. Among qualifying thresholds it chooses the most accepted +examples, breaking ties in favor of the higher threshold. Every required language +must qualify; pooling successful and unsuccessful language slices cannot pass. + +An independent final test allocates `alpha / (2 * languages)` per bound because +each language's threshold is already fixed. Reusing a selection-set ID or related +group is rejected, including reuse across languages. Empty test slices, small +samples, observed errors inconsistent with the budget and insufficient coverage +fail. Deferring everything cannot pass. Never retune a failed policy against the +same final test, repeatedly test candidates until one passes, or omit a failed +language from the protocol; those actions invalidate the declared procedure. + +## Scope of the evidence + +The simultaneous bounds concern independent draws from the evaluated population, +conditional on the frozen upstream model and calibration. They are not guarantees +under future distribution shift, incorrect labels, undetected dependence or a +different sampling scheme. The separate selection and final-test reports each +use their own stated family error probability. Report which experiment a claim +refers to; these are not a combined sequential confidence guarantee. + +The selection report hashes sample identities so the final-test function can +reject reuse. It also records a content hash of the supplied predictions. These +are consistency aids, not authentication: callers must bind the report to the +exact checkpoint, tokenizer, schema, calibration, data manifest and frozen +protocol in a trusted deployment bundle. Matching vector lengths does not verify +label ordering or the model that produced them. This utility cannot verify how +samples were obtained, whether a final set was inspected elsewhere, or whether +the supplied report was replaced. + +Run `python tests/test_risk.py` for offline regression checks. These use synthetic +predictions, published interval references, exact small-sample coverage +enumeration and high-precision tail calculations. They establish statistical +implementation behavior; they do not measure Laya's accuracy, calibration quality +or achievable automation rate on support tickets. diff --git a/laya/risk.py b/laya/risk.py new file mode 100644 index 000000000..37f03ad55 --- /dev/null +++ b/laya/risk.py @@ -0,0 +1,201 @@ +"""Offline selection and independent testing of categorical decision thresholds. + +These opt-in utilities do not change Agent confidence or authorize deployment. +Inputs must be independent representatives from a fixed, audited population. +""" +import hashlib +import json +import math + + +DEFAULT_THRESHOLDS = (0.0, 0.5, 0.6, 0.7, 0.75, 0.8, 0.85, 0.9, 0.925, 0.95, 0.975, 0.99) + + +def _number(value, name, *, low=0.0, high=1.0): + if (isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) + or not low <= value <= high): + raise ValueError("%s must be finite and between %s and %s" % (name, low, high)) + return float(value) + + +def binomial_upper_bound(events: int, trials: int, alpha: float = 0.05) -> float: + """One-sided Clopper-Pearson upper bound, by log-CDF inversion. + + No observations gives 1, not a zero-error claim. Assumes independent Bernoulli + trials. Alpha is the tail probability for this one comparison, not a family. + """ + if type(events) is not int or type(trials) is not int or not 0 <= events <= trials: + raise ValueError("events and trials must be integers with 0 <= events <= trials") + alpha = _number(alpha, "alpha") + if not 0 < alpha < 1: + raise ValueError("alpha must be strictly between zero and one") + if events == trials: + return 1.0 + if events == 0: + return -math.expm1(math.log(alpha) / trials) + coefficients = [math.lgamma(trials + 1) - math.lgamma(j + 1) - math.lgamma(trials - j + 1) + for j in range(events + 1)] + low, high = events / trials, 1.0 + for _ in range(60): + p = (low + high) / 2 + if p == low or p == high: + break + log_p, log_q = math.log(p), math.log1p(-p) + terms = [coefficient + j * log_p + (trials - j) * log_q + for j, coefficient in enumerate(coefficients)] + largest = max(terms) + log_cdf = largest + math.log(math.fsum(math.exp(term - largest) for term in terms)) + if log_cdf > math.log(alpha): + low = p + else: + high = p + return high + + +def _hash(value): + data = json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + return hashlib.sha256(data.encode("utf-8")).hexdigest() + + +def _protocol(languages, max_error, min_coverage, alpha): + if (not isinstance(languages, (list, tuple)) or not languages + or any(not isinstance(lang, str) or not lang.strip() for lang in languages) + or len(set(languages)) != len(languages)): + raise ValueError("required_languages must be a nonempty list of unique language tags") + max_error = _number(max_error, "max_error") + min_coverage = _number(min_coverage, "min_coverage") + alpha = _number(alpha, "alpha") + if max_error >= 1 or min_coverage <= 0 or not 0 < alpha < 1: + raise ValueError("require max_error < 1, min_coverage > 0 and 0 < alpha < 1") + return {"required_languages": sorted(languages), "max_error": max_error, + "min_coverage": min_coverage, "alpha": alpha} + + +def _samples(samples, languages, n_options=None): + """Validate one fixed question's predictions; repeated groups cannot inflate n.""" + rows, ids, groups = [], set(), set() + for sample in samples: + if not isinstance(sample, dict): + raise ValueError("each prediction sample must be a dictionary") + for field in ("id", "group_id", "language"): + if not isinstance(sample.get(field), str) or not sample[field].strip(): + raise ValueError("prediction %s must be a nonempty string" % field) + key, group, lang = sample["id"], sample["group_id"], sample["language"] + if lang not in languages: + raise ValueError("prediction language %r was not declared in the protocol" % lang) + if key in ids or (group, lang) in groups: + raise ValueError("predictions must have unique ids and one independent group per language") + ids.add(key) + groups.add((group, lang)) + probabilities = sample.get("probabilities") + if not isinstance(probabilities, (list, tuple)) or len(probabilities) < 2: + raise ValueError("predictions need a probability vector with at least two options") + probabilities = [_number(value, "probability") for value in probabilities] + if not math.isclose(math.fsum(probabilities), 1.0, rel_tol=0, abs_tol=1e-6): + raise ValueError("probabilities must sum to one") + if n_options is None: + n_options = len(probabilities) + if len(probabilities) != n_options: + raise ValueError("all predictions must have the same fixed option schema") + target = sample.get("target") + if type(target) is not int or not 0 <= target < n_options: + raise ValueError("target must be an integer option index") + prediction = max(range(n_options), key=probabilities.__getitem__) + rows.append({"id": key, "group_id": group, "language": lang, "probabilities": probabilities, + "target": target, "score": probabilities[prediction], "error": int(prediction != target)}) + return sorted(rows, key=lambda row: row["id"]), n_options + + +def _summary(rows, threshold, protocol, tail): + selected = [] if threshold is None else [row for row in rows if row["score"] >= threshold] + n, accepted = len(rows), len(selected) + errors = sum(row["error"] for row in selected) + upper = binomial_upper_bound(errors, accepted, tail) + # A lower bound on successes is one minus an upper bound on failures. + lower = 1 - binomial_upper_bound(n - accepted, n, tail) + qualified = upper <= protocol["max_error"] and lower >= protocol["min_coverage"] + return {"threshold": threshold, "samples": n, "accepted": accepted, "errors": errors, + "coverage": accepted / n if n else 0.0, "error_rate": errors / accepted if accepted else None, + "error_upper": upper, "coverage_lower": lower, "qualified": qualified} + + +def select_risk_policy(samples, *, required_languages, thresholds=DEFAULT_THRESHOLDS, + max_error=0.05, min_coverage=0.5, alpha=0.05): + """Select from a threshold grid fixed before observing selection-set outcomes. + + Scores are max calibrated class probabilities, not the SDK's entropy-based + confidence. Bonferroni covers both bounds, every threshold and every declared + language. Missing evidence or insufficient coverage cannot qualify. + """ + protocol = _protocol(required_languages, max_error, min_coverage, alpha) + if not isinstance(thresholds, (list, tuple)) or not thresholds: + raise ValueError("thresholds must be a nonempty fixed list") + grid = [_number(value, "threshold") for value in thresholds] + if len(set(grid)) != len(grid): + raise ValueError("threshold grid must not contain duplicates") + grid.sort() + rows, n_options = _samples(samples, protocol["required_languages"]) + tail = protocol["alpha"] / (2 * len(required_languages) * len(grid)) + by_language = {} + for lang in protocol["required_languages"]: + subset = [row for row in rows if row["language"] == lang] + candidates = [_summary(subset, threshold, protocol, tail) for threshold in grid] + eligible = [candidate for candidate in candidates if candidate["qualified"]] + chosen = max(eligible, key=lambda row: (row["accepted"], row["threshold"])) if eligible else None + by_language[lang] = {"selected": chosen, "candidates": candidates} + return {"schema_version": 1, "stage": "selection", "protocol": protocol, "threshold_grid": grid, + "n_options": n_options, "tail_probability": tail, "by_language": by_language, + "samples_sha256": _hash(rows), "group_hashes": sorted({_hash(row["group_id"]) for row in rows}), + "id_hashes": sorted({_hash(row["id"]) for row in rows}), + "passed": all(row["selected"] is not None for row in by_language.values())} + + +def evaluate_risk_policy(samples, selection): + """Test a previously selected policy once on fresh independent groups. + + The report concerns the predeclared population and sampling scheme, not future + distribution shifts. Keep this report bound to model, schema and calibration + artifacts externally; these functions do not load or authenticate a bundle. + """ + if (not isinstance(selection, dict) or type(selection.get("schema_version")) is not int + or selection["schema_version"] != 1 or selection.get("stage") != "selection" + or selection.get("passed") is not True): + raise ValueError("independent testing requires a qualifying selection report") + settings = selection.get("protocol") + if not isinstance(settings, dict): + raise ValueError("selection report has no protocol") + try: + protocol = _protocol(settings["required_languages"], settings["max_error"], + settings["min_coverage"], settings["alpha"]) + except KeyError as exc: + raise ValueError("selection protocol is incomplete") from exc + n_options = selection.get("n_options") + if type(n_options) is not int or n_options < 2: + raise ValueError("selection report needs a fixed option count") + by_language = selection.get("by_language") + if not isinstance(by_language, dict) or set(by_language) != set(protocol["required_languages"]): + raise ValueError("selection language slices do not match its protocol") + thresholds = {} + for lang, entry in by_language.items(): + chosen = entry.get("selected") if isinstance(entry, dict) else None + if not isinstance(chosen, dict) or chosen.get("qualified") is not True: + raise ValueError("each required language must have a qualifying selected threshold") + thresholds[lang] = _number(chosen.get("threshold"), "selected threshold") + previous = {} + for field in ("group_hashes", "id_hashes"): + hashes = selection.get(field) + if (not isinstance(hashes, list) or not hashes or any(not isinstance(value, str) or len(value) != 64 + for value in hashes)): + raise ValueError("selection report is missing its sample identities") + previous[field] = set(hashes) + rows, _ = _samples(samples, protocol["required_languages"], n_options) + if (any(_hash(row["group_id"]) in previous["group_hashes"] for row in rows) + or any(_hash(row["id"]) in previous["id_hashes"] for row in rows)): + raise ValueError("final test reuses selection-set examples or related groups") + tail = protocol["alpha"] / (2 * len(protocol["required_languages"])) + results = {lang: _summary([row for row in rows if row["language"] == lang], threshold, protocol, tail) + for lang, threshold in thresholds.items()} + return {"schema_version": 1, "stage": "final_test", "protocol": protocol, "n_options": n_options, + "selection_sha256": _hash(selection), "samples_sha256": _hash(rows), + "tail_probability": tail, "by_language": results, + "passed": all(result["qualified"] for result in results.values())} diff --git a/tests/test_risk.py b/tests/test_risk.py new file mode 100644 index 000000000..bd7f444ca --- /dev/null +++ b/tests/test_risk.py @@ -0,0 +1,191 @@ +"""Selective-risk regression tests on synthetic independent predictions, CPU only.""" +import copy +from decimal import Decimal, localcontext +import math +import os +from pathlib import Path +import sys +import unittest + +os.environ.setdefault("USE_TF", "0") +os.environ.setdefault("USE_TORCH", "1") +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from laya.risk import binomial_upper_bound, evaluate_risk_policy, select_risk_policy # noqa: E402 + + +def samples(prefix, size=300, languages=("en", "de"), error_every=None, score=0.95): + return [{"id": "%s:%s:%s" % (prefix, lang, i), "group_id": "%s:family:%s" % (prefix, i), + "language": lang, "probabilities": [score, 1 - score], + "target": 1 if error_every and i % error_every == 0 else 0} + for lang in languages for i in range(size)] + + +class BinomialTests(unittest.TestCase): + def test_closed_form_boundaries_and_published_reference(self): + self.assertEqual(binomial_upper_bound(0, 0), 1) + self.assertEqual(binomial_upper_bound(100, 100), 1) + self.assertAlmostEqual(binomial_upper_bound(0, 100), 1 - 0.05 ** 0.01, places=14) + self.assertAlmostEqual(binomial_upper_bound(1, 2), math.sqrt(0.95), places=14) + # SciPy's published exact 95% two-sided interval for 7/50 uses alpha=.025 + # in its upper tail: docs.scipy.org/.../BinomTestResult.proportion_ci.html + self.assertAlmostEqual(binomial_upper_bound(7, 50, 0.025), 0.26739600249700846, places=12) + + def test_exact_small_sample_coverage(self): + # Enumerate the binomial sampling distribution, rather than simulating a + # conveniently passing seed. The probability of undercoverage is <= alpha. + for n in (1, 2, 5, 10, 25): + for alpha in (0.01, 0.05, 0.2): + bounds = [binomial_upper_bound(k, n, alpha) for k in range(n + 1)] + self.assertEqual(bounds, sorted(bounds)) + for step in range(1, 100): + p = step / 100 + missed = math.fsum(math.comb(n, k) * p ** k * (1 - p) ** (n - k) + for k in range(n + 1) if bounds[k] < p) + self.assertLessEqual(missed, alpha + 1e-12) + + def test_invalid_counts_and_alpha(self): + for k, n, alpha in [(True, 1, .05), (1.0, 2, .05), (-1, 2, .05), (2, 1, .05), + (0, -1, .05), (0, 1, 0), (0, 1, 1), (0, 1, float("nan"))]: + with self.subTest(k=k, n=n, alpha=alpha), self.assertRaises(ValueError): + binomial_upper_bound(k, n, alpha) + + def test_large_sample_tail_against_decimal_probability_recurrence(self): + # Independent arithmetic checks log-gamma cancellation and extreme tails. + with localcontext() as context: + context.prec = 60 + for k, n, alpha in [(100, 5000, .001), (990, 1000, .05), (1, 10000, 1e-8)]: + bound = binomial_upper_bound(k, n, alpha) + p = Decimal(str(bound)) + q = 1 - p + term = q ** n + cdf = term + for j in range(1, k + 1): + term *= Decimal(n - j + 1) / j * p / q + cdf += term + self.assertAlmostEqual(float(cdf) / alpha, 1.0, places=8) + + +class SelectionTests(unittest.TestCase): + def test_correct_confident_population_can_pass_and_roundtrip_json(self): + import json + rows = samples("policy") + before = copy.deepcopy(rows) + selected = select_risk_policy(rows, required_languages=["de", "en"], thresholds=[.5, .9, .99]) + self.assertTrue(selected["passed"]) + self.assertEqual(selected["tail_probability"], .05 / (2 * 2 * 3)) + self.assertEqual(selected["by_language"]["en"]["selected"]["threshold"], .9) + self.assertEqual(rows, before) + self.assertEqual(selected, select_risk_policy(reversed(rows), required_languages=["en", "de"], + thresholds=[.99, .9, .5])) + tested = evaluate_risk_policy(samples("test"), json.loads(json.dumps(selected))) + self.assertTrue(tested["passed"]) + self.assertEqual(tested["tail_probability"], .05 / (2 * 2)) + self.assertGreater(tested["by_language"]["en"]["error_upper"], 0) + self.assertLess(tested["by_language"]["en"]["coverage_lower"], 1) + + def test_overconfident_errors_do_not_pass(self): + selected = select_risk_policy(samples("wrong", error_every=2, score=1), required_languages=["en", "de"]) + self.assertFalse(selected["passed"]) + self.assertTrue(all(entry["selected"] is None for entry in selected["by_language"].values())) + + def test_selection_excludes_uncertain_errors_while_retaining_coverage(self): + rows = samples("mixed", size=1000, languages=("en",)) + for i, row in enumerate(rows): + if i >= 700: + row["probabilities"] = [.6, .4] + row["target"] = 1 + elif i % 100 == 0: + row["target"] = 1 + selected = select_risk_policy(rows, required_languages=["en"], thresholds=[.5, .9, .99]) + chosen = selected["by_language"]["en"]["selected"] + self.assertTrue(selected["passed"]) + self.assertEqual(chosen["threshold"], .9) + self.assertEqual(chosen["accepted"], 700) + self.assertEqual(chosen["errors"], 7) + self.assertGreater(chosen["error_upper"], .01) + self.assertGreaterEqual(chosen["coverage_lower"], .5) + + def test_more_threshold_comparisons_require_more_evidence(self): + rows = samples("limited", size=80, languages=("en",)) + single = select_risk_policy(rows, required_languages=["en"], thresholds=[.9]) + multiple = select_risk_policy(rows, required_languages=["en"], thresholds=[.8, .9, .95]) + self.assertTrue(single["passed"]) + self.assertFalse(multiple["passed"]) + + def test_no_evidence_and_too_few_examples_cannot_pass(self): + for rows in ([], samples("small", size=5), samples("missing", languages=("en",))): + with self.subTest(size=len(rows)): + selected = select_risk_policy(rows, required_languages=["en", "de"]) + self.assertFalse(selected["passed"]) + with self.assertRaises(ValueError): + evaluate_risk_policy(samples("test"), selected) + + def test_defer_everything_fails_coverage_and_has_no_error_estimate(self): + selected = select_risk_policy(samples("defer", score=.5), required_languages=["en", "de"], thresholds=[.9]) + result = selected["by_language"]["en"]["candidates"][0] + self.assertFalse(selected["passed"]) + self.assertEqual(result["accepted"], 0) + self.assertEqual(result["coverage_lower"], 0) + self.assertEqual(result["error_upper"], 1) + self.assertIsNone(result["error_rate"]) + + def test_language_failure_cannot_be_hidden_by_pooling(self): + rows = samples("good", size=1000, languages=("en",)) + samples("bad", size=100, languages=("de",), error_every=2) + selected = select_risk_policy(rows, required_languages=["en", "de"], thresholds=[.9]) + self.assertFalse(selected["passed"]) + self.assertIsNotNone(selected["by_language"]["en"]["selected"]) + self.assertIsNone(selected["by_language"]["de"]["selected"]) + + def test_duplicates_cannot_inflate_evidence(self): + rows = samples("repeat", size=1, languages=("en",)) + duplicate = {**rows[0], "id": "different-id"} + for extra in (rows[0], duplicate): + with self.assertRaisesRegex(ValueError, "independent group"): + select_risk_policy(rows + [extra], required_languages=["en"]) + + def test_bad_predictions_and_protocols_are_rejected(self): + for fields in ({"probabilities": [1.2, -.2]}, {"probabilities": [.9, .9]}, + {"probabilities": [float("nan"), 0]}, {"probabilities": [True, 0]}, + {"probabilities": [1]}, {"target": True}, {"target": 2}, + {"target": "0"}, {"group_id": ""}, {"language": "fr"}): + row = {**samples("invalid", size=1, languages=("en",))[0], **fields} + with self.subTest(fields=fields), self.assertRaises(ValueError): + select_risk_policy([row], required_languages=["en"]) + for settings in ({"required_languages": []}, {"required_languages": ["en", "en"]}, + {"min_coverage": 0}, {"max_error": 1}, {"alpha": 1}, + {"thresholds": []}, {"thresholds": [.9, .9]}, {"thresholds": [float("inf")]}): + with self.subTest(settings=settings), self.assertRaises(ValueError): + select_risk_policy([], **{"required_languages": ["en"], **settings}) + + +class IndependentTestTests(unittest.TestCase): + def setUp(self): + self.selection = select_risk_policy(samples("policy"), required_languages=["en", "de"], thresholds=[.9]) + + def test_selection_examples_and_families_cannot_be_reused(self): + for field, value in [("id", "policy:en:0"), ("group_id", "policy:family:0")]: + rows = samples("test") + rows[0][field] = value + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "reuses"): + evaluate_risk_policy(rows, self.selection) + + def test_locked_policy_can_fail_fresh_population(self): + for rows in ([], samples("test", score=.6), samples("test", error_every=3)): + with self.subTest(size=len(rows)): + self.assertFalse(evaluate_risk_policy(rows, self.selection)["passed"]) + + def test_option_count_and_incomplete_selection_are_rejected(self): + rows = samples("test") + rows[0]["probabilities"] = [.9, .05, .05] + with self.assertRaisesRegex(ValueError, "fixed option schema"): + evaluate_risk_policy(rows, self.selection) + for field, value in [("schema_version", True), ("stage", "final_test"), ("group_hashes", []), + ("id_hashes", None), ("n_options", True), ("by_language", {})]: + invalid = {**self.selection, field: value} + with self.subTest(field=field), self.assertRaises(ValueError): + evaluate_risk_policy(samples("test"), invalid) + + +if __name__ == "__main__": + unittest.main()