diff --git a/README.md b/README.md index 16744f965..9105b17ed 100644 --- a/README.md +++ b/README.md @@ -415,6 +415,10 @@ result["shortlist"]["intent"]["labels"] # the top 20 labels sent to the model ## Fine-Tuning +For custom labelled examples, the opt-in [adaptation data utilities](docs/adaptation_data.md) +validate typed labels, track provenance and keep related examples together in four +reproducible partitions for training, calibration, policy selection and final testing. + Fine-tune Laya on your own domain data. The notebook runs on Kaggle's free 2xT4 GPUs and does the whole loop: build the dataset, train with RLCD (proper-scoring-rule rewards, GRPO-style policy gradient), fit calibration temperatures, evaluate, and push the result to the Hub. diff --git a/docs/adaptation_data.md b/docs/adaptation_data.md new file mode 100644 index 000000000..719256afc --- /dev/null +++ b/docs/adaptation_data.md @@ -0,0 +1,128 @@ +# Labelled datasets for domain adaptation + +`laya.adapt_data` prepares local labelled examples with a reproducible split +manifest. It does not download a checkpoint, train a model, fit calibration or +authorize automated decisions. The existing prediction API is unchanged. + +Use the same question definitions as `Agent.predict`. Gold labels are strict: + +| Question type | Gold target | +| --- | --- | +| `choice` | An exact criterion label, not its integer position | +| `score` | A zero-based integer rubric index; booleans and floats are rejected | +| `noul` | A JSON boolean, `true` or `false` | + +An example can label only a subset of the questions. It must have at least one +label, and cannot name an unknown question. Structured JSON instructions and +criterion descriptions are supported. Question order, option order and structured +text field order are included in fingerprints because they can affect model input. + +```python +from laya.adapt_data import read_dataset, write_dataset + +questions = { + "queue": { + "type": "choice", + "instructions": "Which queue should handle this ticket?", + "criteria": {"billing": "payments and invoices", "technical": "bugs and outages"}, + } +} +examples = [{ + "id": "example:1", + "group_id": "example:conversation:1", + "language": "en", + "state": {"subject": "Invoice", "body": "I was charged twice."}, + "targets": {"queue": "billing"}, + "source": { + "dataset": "local-example", + "revision": "v1", + "license": "CC0-1.0", + "record_id": "1", + }, +}] + +# Choose a new output directory; existing datasets are never updated in place. +manifest = write_dataset("prepared-tickets", examples, questions, seed="tickets-v1") +questions, examples, manifest = read_dataset("prepared-tickets") +print(manifest["counts"]) # One example is insufficient for four usable partitions. +``` + +Replace the illustrative provenance with the source's actual dataset identifier, +immutable revision (or content hash), license and original record ID. Recording a +license does not verify permissions. Do not put gold labels, answer drafts or +other information unavailable at inference time in `state`. Language tags must be +lowercase (for example `en`, `de` or `pt-br`); they record the source assertion and +are not independently verified language detection. + +## Grouping and independence + +Assign a globally unique `group_id` to each conversation or related example +family. Translations, paraphrases and augmentations of the same example must +share this group. The utility additionally joins groups connected by either: + +- The same source dataset and original record ID, including across revisions. +- Identical state after Unicode NFC and whitespace normalization, ignoring JSON + object key order for duplicate detection only. Case and list order are preserved. + +Connections are transitive. Contradictory labels for duplicate states fail +validation. Matching finds normalized exact duplicates; it does not discover +unmarked translations or semantic near-duplicates. Source adapters remain +responsible for preserving those relationships and auditing their data. + +Each connected group is hashed with the seed into one of four roles: `train` +(60%), `calibration` (15%), `policy` (10%) and `test` (15%). Fractions can be +overridden with four nonnegative values summing to one. Assignment is independent of +input row order and uses no gold labels. Fractions are expected proportions, not +guaranteed split sizes or stratification. Small or skewed datasets can leave a +partition, task or language without enough evidence; inspect the counts before +training. Adding a connecting record can merge groups and change assignments, +so freeze the entire prepared dataset before running an experiment. + +To preserve a source's official test set, pass `fixed_splits={"example:42": "test"}` +to `write_dataset` or `make_split_manifest`. A reserved example places its entire +connected family in that role; contradictory reservations are rejected. The +manifest records and verifies reservations. Fractions then apply only to the +remaining groups. Setting the test fraction to zero reserves that role exclusively +for the fixed test families. Missing roles still provide no usable evidence. + +The manifest retains **all rows**, including duplicates, and reports both row +counts and connected-group counts by split and language. Neither retaining rows +nor placing families in separate splits makes repeated observations independent. +`select_independent_records(examples, questions, manifest, split="policy", +question_id="queue")` selects one labelled representative per connected group and +language. It uses the frozen seed, question ID and example ID, never target values +or model predictions. Apply it before evaluating a task. Missing task labels are +excluded; a partition without that label returns no evidence. Other questions can +select different representatives. These estimates describe this sampling scheme, +not the frequency-weighted population of all raw rows. Independence still depends +on the adapter correctly identifying families. Never use duplicate row counts as +binomial sample sizes. + +Use training data for fitting weights, calibration data for temperatures, policy +data for selecting review thresholds, and test data only for a locked final +evaluation. This module records those roles; downstream runners must enforce +access to the appropriate partition. A manifest is not an access-control system. + +## Persistence and verification + +The prepared directory contains `questions.json`, `records.jsonl` and +`splits.json`. Rows are sorted by ID. UTF-8 and LF line endings give portable +files, while JSON state and question ordering survive a round trip. Publication +stages files beside the destination and renames the complete directory; a failed +write does not publish a partial dataset. Use a unique destination per publisher; +concurrent publication to the same path is not supported. + +`read_dataset` revalidates labels, source fields, grouping, assignments, counts and +fingerprints. Data or schema changes that make the manifest inconsistent are +rejected. This is an integrity check, not authentication: an actor who can replace +all files can recompute a valid manifest. Experiment and deployment artifacts +should bind the hash of the frozen manifest from a separately trusted record. + +Run the synthetic CPU-only contract tests with: + +```shell +python tests/test_adapt_data.py +``` + +These checks establish data handling behavior. They do not measure prediction +accuracy, calibration quality or an achievable automation rate. diff --git a/laya/adapt_data.py b/laya/adapt_data.py new file mode 100644 index 000000000..2b1d5073f --- /dev/null +++ b/laya/adapt_data.py @@ -0,0 +1,340 @@ +"""Validated labelled examples and reproducible group splits for Laya adaptation. + +Opt-in dataset utilities. No model loading, downloads or training happen here. +""" +import copy +import hashlib +import json +import math +from pathlib import Path +import re +import tempfile +import unicodedata +from typing import Any, Dict, Iterable, Mapping + + +SCHEMA_VERSION = 1 +SPLITS = ("train", "calibration", "policy", "test") +DEFAULT_FRACTIONS = {"train": 0.6, "calibration": 0.15, "policy": 0.1, "test": 0.15} +_LANGUAGE = re.compile(r"[a-z]{2,3}(?:-[a-z0-9]{2,8})*\Z") + + +def canonical_json(value: Any) -> str: + """Stable JSON for metadata fingerprints; refuse non-finite numbers.""" + return json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False, separators=(",", ":")) + + +def fingerprint(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError("%s must be a nonempty string" % field) + return value + + +def _json_value(value: Any, field: str) -> None: + # Reject non-string keys rather than allowing JSON to silently coerce them. + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError("%s must have string object keys" % field) + _json_value(item, field) + elif isinstance(value, list): + for item in value: + _json_value(item, field) + elif value is not None and not isinstance(value, (str, bool, int, float)): + raise ValueError("%s must contain only JSON values" % field) + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("%s must not contain NaN or infinity" % field) + + +def validate_questions(questions: Mapping) -> Dict[str, Dict]: + """Copy and validate the existing SDK's typed question schema for training.""" + if not isinstance(questions, dict) or not questions: + raise ValueError("questions must be a nonempty dictionary") + _json_value(questions, "questions") + for qid, question in questions.items(): + _text(qid, "question id") + if not isinstance(question, dict): + raise ValueError("question %r must be a dictionary" % qid) + kind = question.get("type") + if kind not in ("choice", "score", "noul"): + raise ValueError("question %r has an unsupported type" % qid) + if "instructions" not in question: + raise ValueError("question %r is missing instructions" % qid) + criteria = question.get("criteria") + if kind == "choice": + if not isinstance(criteria, (dict, list)) or not criteria: + raise ValueError("choice %r must have nonempty dict/list criteria" % qid) + labels = list(criteria) + for label in labels: + _text(label, "choice label") + if len(set(labels)) != len(labels): + raise ValueError("choice %r has duplicate labels" % qid) + elif kind == "score": + if not isinstance(criteria, list) or not criteria: + raise ValueError("score %r must have a nonempty list of levels" % qid) + elif criteria is not None: + if not isinstance(criteria, dict) or set(criteria) - {"false", "true"}: + raise ValueError("noul %r criteria must use false/true string keys" % qid) + return copy.deepcopy(questions) + + +def question_fingerprint(questions: Mapping) -> str: + """Preserve all object ordering: structured instructions also render as text.""" + checked = validate_questions(questions) + return fingerprint(json.dumps(checked, ensure_ascii=False, allow_nan=False, separators=(",", ":"))) + + +def _records_fingerprint(rows: list) -> str: + # Metadata key order has no effect, but state object order affects SDK text. + return fingerprint([{**row, "state": json.dumps(row["state"], ensure_ascii=False, + allow_nan=False, separators=(",", ":"))} for row in rows]) + + +def target_index(question: Mapping, target: Any) -> int: + """Convert a hard gold label to the SDK's option index without coercion.""" + kind = question["type"] + if kind == "choice": + labels = list(question["criteria"]) + if not isinstance(target, str) or target not in labels: + raise ValueError("choice target must be one of the question's labels") + return labels.index(target) + if kind == "noul": + if not isinstance(target, bool): + raise ValueError("noul target must be a JSON boolean") + return int(target) + if kind == "score": + if isinstance(target, bool) or not isinstance(target, int) or not 0 <= target < len(question["criteria"]): + raise ValueError("score target must be an integer index of a rubric level") + return target + raise ValueError("unsupported question type") + + +def validate_records(records: Iterable[Mapping], questions: Mapping) -> list: + """Validate hard-labelled examples and provenance, preserving caller objects. + + Required fields: id, group_id, language, state, targets, source. Source records + dataset/revision/license/record_id. Related translations, conversations and + augmentations must share a globally unique group_id. Source identities and + normalized exact duplicates are also linked automatically. Missing task labels + are allowed, but unknown labels are not. A language tag is a source assertion, + not a verified language detection result. + """ + checked = validate_questions(questions) + result, seen = [], set() + for row in records: + if not isinstance(row, dict): + raise ValueError("each example must be a dictionary") + _json_value(row, "example") + key = _text(row.get("id"), "example id") + if key in seen: + raise ValueError("duplicate example id %r" % key) + seen.add(key) + _text(row.get("group_id"), "group_id for %r" % key) + language = row.get("language") + if not isinstance(language, str) or not _LANGUAGE.fullmatch(language): + raise ValueError("example %r needs a lowercase language tag, e.g. en or de" % key) + if not isinstance(row.get("state"), (str, dict, list)): + raise ValueError("example %r state must be text, a JSON object or a list" % key) + targets = row.get("targets") + if not isinstance(targets, dict) or not targets or set(targets) - set(checked): + raise ValueError("example %r needs targets for known question ids" % key) + for qid, target in targets.items(): + try: + target_index(checked[qid], target) + except ValueError as exc: + raise ValueError("example %r, question %r: %s" % (key, qid, exc)) from exc + source = row.get("source") + if not isinstance(source, dict): + raise ValueError("example %r needs source provenance" % key) + for field in ("dataset", "revision", "license", "record_id"): + _text(source.get(field), "source.%s for %r" % (field, key)) + result.append(copy.deepcopy(row)) + if not result: + raise ValueError("dataset must contain at least one labelled example") + return sorted(result, key=lambda row: row["id"]) + + +def _normalized_state(value: Any) -> Any: + if isinstance(value, str): + return " ".join(unicodedata.normalize("NFC", value).split()) + if isinstance(value, list): + return [_normalized_state(item) for item in value] + if isinstance(value, dict): + return {key: _normalized_state(item) for key, item in value.items()} + return value + + +def make_split_manifest(records: Iterable[Mapping], questions: Mapping, *, seed: str = "laya-v1", + fractions: Mapping = None, fixed_splits: Mapping = None) -> Dict: + """Hash related groups into four partitions, independent of input row order. + + Groups connected by normalized duplicate states or source identities are merged + transitively. Contradictory labels for duplicate states are rejected. Row counts + include duplicates and must not be used as independent statistical sample sizes. + Fractions apply to unreserved groups. fixed_splits optionally reserves record + IDs for named partitions and propagates that reservation to their whole family. + Conflicting reservations fail. Small splits can be empty. + """ + rows = validate_records(records, questions) + _text(seed, "split seed") + fractions = dict(DEFAULT_FRACTIONS if fractions is None else fractions) + if set(fractions) != set(SPLITS) or any( + isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v < 0 + for v in fractions.values() + ) or not math.isclose(sum(fractions.values()), 1.0, rel_tol=0, abs_tol=1e-12): + raise ValueError("split fractions must be nonnegative train/calibration/policy/test values summing to one") + if fixed_splits is not None: + if (not isinstance(fixed_splits, dict) or set(fixed_splits) - {row["id"] for row in rows} + or any(name not in SPLITS for name in fixed_splits.values())): + raise ValueError("fixed_splits must map known example ids to named partitions") + parent = {row["group_id"]: row["group_id"] for row in rows} + + def root(group): + while parent[group] != group: + parent[group] = parent[parent[group]] + group = parent[group] + return group + + def merge(a, b): + first, second = sorted((root(a), root(b))) + parent[second] = first + + duplicate_states, source_groups, duplicate_count = {}, {}, 0 + for row in rows: + # Different revisions of the same source record remain related even if its + # text has changed. Adapters must retain the original identity for variants. + source = row["source"] + identity = (source["dataset"], source["record_id"]) + if identity in source_groups: + merge(source_groups[identity], row["group_id"]) + else: + source_groups[identity] = row["group_id"] + state_hash = fingerprint(_normalized_state(row["state"])) + if state_hash in duplicate_states: + first = duplicate_states[state_hash] + shared = set(first["targets"]) & set(row["targets"]) + if any(first["targets"][qid] != row["targets"][qid] for qid in shared): + raise ValueError("contradictory duplicate labels in examples %r and %r" % (first["id"], row["id"])) + first["targets"].update(row["targets"]) + merge(first["group_id"], row["group_id"]) + duplicate_count += 1 + else: + # A private copy accumulates partial labels without changing dataset rows. + duplicate_states[state_hash] = copy.deepcopy(row) + reserved = {} + for row in rows: + if fixed_splits is not None and row["id"] in fixed_splits: + group, role = root(row["group_id"]), fixed_splits[row["id"]] + if group in reserved and reserved[group] != role: + raise ValueError("related examples have conflicting fixed split reservations") + reserved[group] = role + assignments, groups = {}, {} + counts = {name: 0 for name in SPLITS} + languages = {name: {} for name in SPLITS} + group_sets = {name: set() for name in SPLITS} + language_group_sets = {name: {} for name in SPLITS} + for row in rows: + group = root(row["group_id"]) + value = int(fingerprint([seed, group]), 16) / 2 ** 256 + # Rounding a 256-bit hash to float can produce 1.0; never fall back to a + # zero-probability partition, including one reserved for official tests. + bound = 0.0 + chosen = next(name for name in reversed(SPLITS) if fractions[name] > 0) + for name in SPLITS: + bound += fractions[name] + if value < bound: + chosen = name + break + chosen = reserved.get(group, chosen) + assignments[row["id"]] = chosen + groups[row["id"]] = group + counts[chosen] += 1 + group_sets[chosen].add(group) + lang = row["language"] + languages[chosen][lang] = languages[chosen].get(lang, 0) + 1 + language_group_sets[chosen].setdefault(lang, set()).add(group) + manifest = { + "schema_version": SCHEMA_VERSION, "seed": seed, + "fractions": {name: fractions[name] for name in SPLITS}, + "questions_sha256": question_fingerprint(questions), "records_sha256": _records_fingerprint(rows), + "assignments": assignments, "groups": groups, "counts": counts, "languages": languages, + "group_counts": {name: len(groups) for name, groups in group_sets.items()}, + "language_group_counts": {name: {lang: len(groups) for lang, groups in languages.items()} + for name, languages in language_group_sets.items()}, + "normalized_duplicate_rows": duplicate_count, "group_count": len(set(groups.values())), + } + if fixed_splits is not None: + manifest["fixed_splits"] = dict(sorted(fixed_splits.items())) + return manifest + + +def verify_split_manifest(records: Iterable[Mapping], questions: Mapping, manifest: Mapping) -> None: + if (not isinstance(manifest, dict) or type(manifest.get("schema_version")) is not int + or manifest["schema_version"] != SCHEMA_VERSION): + raise ValueError("unsupported split manifest schema") + expected = make_split_manifest(records, questions, seed=manifest.get("seed"), fractions=manifest.get("fractions"), + fixed_splits=manifest.get("fixed_splits")) + # JSON comparison also rejects bools masquerading as integer counts in Python. + if canonical_json(manifest) != canonical_json(expected): + raise ValueError("split manifest does not match the data, questions or recorded split configuration") + + +def select_independent_records(records: Iterable[Mapping], questions: Mapping, manifest: Mapping, *, + split: str, question_id: str) -> list: + """Select one labelled example per connected group and language for a task. + + Selection uses IDs and the frozen split seed, never target values or model + scores. Apply before evaluation; row-level error bounds on all family members + would incorrectly treat dependent examples as independent evidence. + """ + rows = validate_records(records, questions) + verify_split_manifest(rows, questions, manifest) + if split not in SPLITS or question_id not in questions: + raise ValueError("select a known split and question id") + selected = {} + for row in rows: + if manifest["assignments"][row["id"]] != split or question_id not in row["targets"]: + continue + key = (manifest["groups"][row["id"]], row["language"]) + rank = (fingerprint([manifest["seed"], question_id, row["id"]]), row["id"]) + if key not in selected or rank < selected[key][0]: + selected[key] = (rank, row) + return sorted((row for _, row in selected.values()), key=lambda row: row["id"]) + + +def write_dataset(destination, records: Iterable[Mapping], questions: Mapping, *, seed="laya-v1", + fractions=None, fixed_splits=None) -> Dict: + """Publish a new dataset directory atomically; never overwrite an existing one.""" + rows = validate_records(records, questions) + checked = validate_questions(questions) + manifest = make_split_manifest(rows, checked, seed=seed, fractions=fractions, fixed_splits=fixed_splits) + destination = Path(destination) + if destination.exists(): + raise FileExistsError("dataset destination already exists: %s" % destination) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".laya-dataset-", dir=destination.parent) as temp: + staged = Path(temp) / "dataset" + staged.mkdir() + # Preserve question/choice order; the fingerprint explicitly includes it. + (staged / "questions.json").write_text(json.dumps(checked, ensure_ascii=False, allow_nan=False, indent=2) + "\n", + encoding="utf-8", newline="\n") + (staged / "records.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False, allow_nan=False, + separators=(",", ":")) + "\n" for row in rows), + encoding="utf-8", newline="\n") + (staged / "splits.json").write_text(canonical_json(manifest) + "\n", encoding="utf-8", newline="\n") + staged.rename(destination) + return manifest + + +def read_dataset(directory): + """Read and verify a prepared dataset, returning questions, rows and manifest.""" + directory = Path(directory) + questions = json.loads((directory / "questions.json").read_text(encoding="utf-8")) + rows = [json.loads(line) for line in (directory / "records.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()] + manifest = json.loads((directory / "splits.json").read_text(encoding="utf-8")) + verify_split_manifest(rows, questions, manifest) + return validate_questions(questions), validate_records(rows, questions), manifest diff --git a/tests/test_adapt_data.py b/tests/test_adapt_data.py new file mode 100644 index 000000000..3e18613a5 --- /dev/null +++ b/tests/test_adapt_data.py @@ -0,0 +1,287 @@ +"""Offline adaptation data contract tests; no model weights or training. + +Run: python tests/test_adapt_data.py +""" +import copy +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +os.environ.setdefault("USE_TF", "0") +os.environ.setdefault("USE_TORCH", "1") +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from laya.adapt_data import ( # noqa: E402 + DEFAULT_FRACTIONS, SPLITS, make_split_manifest, question_fingerprint, + read_dataset, select_independent_records, target_index, validate_questions, validate_records, + verify_split_manifest, write_dataset, +) + + +def questions(): + return { + "queue": {"type": "choice", "instructions": "Route the ticket", + "criteria": {"billing": "payments", "technical": "bugs"}}, + "urgency": {"type": "score", "instructions": "Rate urgency", "criteria": ["low", "high"]}, + "incident": {"type": "noul", "instructions": "Does this report an incident?"}, + } + + +def record(key, **changes): + row = {"id": key, "group_id": "fixture:" + key, "language": "en", "state": "Ticket " + key, + "targets": {"queue": "billing", "urgency": 0, "incident": False}, + "source": {"dataset": "local-synthetic", "revision": "v1", "license": "CC0-1.0", "record_id": key}} + row.update(changes) + return row + + +class ValidationTests(unittest.TestCase): + def test_typed_targets_and_order(self): + q = questions() + for qid, gold, expected in [("queue", "technical", 1), ("urgency", 1, 1), + ("incident", False, 0), ("incident", True, 1)]: + self.assertEqual(target_index(q[qid], gold), expected) + self.assertEqual(validate_records([record("b"), record("a", targets={"queue": "billing"})], q)[0]["id"], "a") + q["queue"]["criteria"] = ["technical", "billing"] + self.assertEqual(target_index(q["queue"], "billing"), 1) + + def test_invalid_targets_are_not_coerced(self): + for qid, gold in [("queue", 0), ("queue", "unknown"), ("urgency", True), ("urgency", 0.0), + ("urgency", -1), ("urgency", 2), ("incident", 1), ("incident", "false")]: + with self.subTest(qid=qid, gold=gold), self.assertRaises(ValueError): + validate_records([record("a", targets={qid: gold})], questions()) + + def test_malformed_records(self): + cases = [{"id": " "}, {"group_id": None}, {"language": "EN"}, {"language": ""}, + {"state": None}, {"state": True}, {"state": {1: "bad key"}}, + {"targets": {}}, {"targets": {"missing": "x"}}, {"source": {}}, + {"metadata": float("nan")}, {"metadata": float("inf")}, {"metadata": object()}] + for changes in cases: + with self.subTest(changes=changes), self.assertRaises(ValueError): + validate_records([record("a", **changes)], questions()) + for rows in ([], [None], [record("a"), record("a")]): + with self.subTest(rows=rows), self.assertRaises(ValueError): + validate_records(rows, questions()) + for field in ("dataset", "revision", "license", "record_id"): + row = record("a") + del row["source"][field] + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "source." + field): + validate_records([row], questions()) + + def test_malformed_questions(self): + cases = [{}, {"q": None}, {"q": {"type": "other", "instructions": "x"}}, + {"q": {"type": "noul"}}, {"q": {"type": "choice", "instructions": "x", "criteria": []}}, + {"q": {"type": "choice", "instructions": "x", "criteria": ["a", "a"]}}, + {"q": {"type": "choice", "instructions": "x", "criteria": [1]}}, + {"q": {"type": "score", "instructions": "x", "criteria": {"low": 0}}}, + {"q": {"type": "score", "instructions": "x", "criteria": []}}, + {"q": {"type": "noul", "instructions": "x", "criteria": {"True": "yes"}}}] + for q in cases: + with self.subTest(q=q), self.assertRaises(ValueError): + validate_questions(q) + + def test_fingerprints_preserve_rendered_order(self): + q = questions() + original = question_fingerprint(q) + q["queue"]["criteria"] = dict(reversed(list(q["queue"]["criteria"].items()))) + self.assertNotEqual(original, question_fingerprint(q)) + q = questions() + q["urgency"]["criteria"].reverse() + self.assertNotEqual(original, question_fingerprint(q)) + self.assertNotEqual(original, question_fingerprint(dict(reversed(list(questions().items()))))) + q["queue"]["instructions"] = {"first": "x", "second": "y"} + structured = question_fingerprint(q) + q["queue"]["instructions"] = {"second": "y", "first": "x"} + self.assertNotEqual(structured, question_fingerprint(q)) + + def test_callers_objects_are_not_mutated(self): + rows = [record("b"), record("a")] + q, before = questions(), copy.deepcopy(rows) + make_split_manifest(rows, q) + checked = validate_records(rows, q) + checked[0]["targets"]["queue"] = "technical" + checked_q = validate_questions(q) + checked_q["queue"]["criteria"]["extra"] = "x" + self.assertEqual(rows, before) + self.assertEqual(q, questions()) + + +class SplitTests(unittest.TestCase): + def test_reproducible_group_assignments(self): + rows = [record(str(i), group_id="family:%d" % (i // 2), language="en" if i % 2 else "de") for i in range(400)] + manifest = make_split_manifest(rows, questions()) + self.assertEqual(manifest, make_split_manifest(reversed(rows), questions())) + self.assertNotEqual(manifest["assignments"], make_split_manifest(rows, questions(), seed="another")["assignments"]) + self.assertTrue(all(manifest["counts"].values())) + for i in range(0, 400, 2): + self.assertEqual(manifest["assignments"][str(i)], manifest["assignments"][str(i + 1)]) + self.assertEqual(sum(manifest["counts"].values()), 400) + self.assertEqual(sum(manifest["group_counts"].values()), 200) + for split in SPLITS: + self.assertEqual(manifest["language_group_counts"][split]["de"], manifest["group_counts"][split]) + self.assertEqual(manifest["language_group_counts"][split]["en"], manifest["group_counts"][split]) + + def test_transitive_duplicates_and_families_stay_together(self): + rows = [record("a", state={"body": "café\n charge"}), + record("b", state={"body": "cafe\u0301 charge"}, group_id="shared"), + record("c", state={"body": "other"}, group_id="shared"), + record("d", state={"body": "other"})] + manifest = make_split_manifest(rows, questions()) + self.assertEqual(len(set(manifest["assignments"].values())), 1) + self.assertEqual(manifest["group_count"], 1) + self.assertEqual(manifest["normalized_duplicate_rows"], 2) + self.assertEqual(sum(manifest["counts"].values()), 4) + + def test_duplicate_source_identity_links_revisions(self): + a, b = record("a"), record("b") + b["source"].update(record_id="a", revision="v2") + manifest = make_split_manifest([a, b], questions()) + self.assertEqual(manifest["groups"]["a"], manifest["groups"]["b"]) + + def test_different_datasets_do_not_link_by_record_id_alone(self): + a, b = record("a"), record("b") + b["source"].update(record_id="a", dataset="another-source") + self.assertEqual(make_split_manifest([a, b], questions())["group_count"], 2) + + def test_duplicate_label_conflicts_including_partial_labels(self): + rows = [record("a", state="same", targets={"queue": "billing"}), + record("b", state="same", targets={"incident": False}), + record("c", state="same", targets={"incident": True})] + before = copy.deepcopy(rows) + with self.assertRaisesRegex(ValueError, "contradictory duplicate"): + make_split_manifest(rows, questions()) + self.assertEqual(rows, before) + rows.pop() + self.assertEqual(make_split_manifest(rows, questions())["normalized_duplicate_rows"], 1) + + def test_small_splits_are_reported_without_duplicating_rows(self): + manifest = make_split_manifest([record("a")], questions()) + self.assertEqual(sorted(manifest["counts"].values()), [0, 0, 0, 1]) + + def test_reserved_source_test_split_propagates_to_related_examples(self): + rows = [record("a", group_id="translation"), record("b", group_id="translation"), record("c")] + fractions = {"train": .7, "calibration": .15, "policy": .15, "test": 0} + manifest = make_split_manifest(rows, questions(), fractions=fractions, fixed_splits={"a": "test"}) + self.assertEqual(manifest["assignments"]["a"], "test") + self.assertEqual(manifest["assignments"]["b"], "test") + self.assertNotEqual(manifest["assignments"]["c"], "test") + verify_split_manifest(rows, questions(), manifest) + with self.assertRaisesRegex(ValueError, "conflicting"): + make_split_manifest(rows, questions(), fixed_splits={"a": "test", "b": "train"}) + for fixed in ({"unknown": "test"}, {"a": "unknown"}): + with self.assertRaises(ValueError): + make_split_manifest(rows, questions(), fixed_splits=fixed) + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) / "reserved" + write_dataset(output, rows, questions(), fractions=fractions, fixed_splits={"a": "test"}) + self.assertEqual(read_dataset(output)[2], manifest) + + def test_rounded_hash_does_not_select_zero_fraction_partition(self): + fractions = {"train": .7, "calibration": .15, "policy": .15, "test": 0} + with patch("laya.adapt_data.fingerprint", return_value="f" * 64): + manifest = make_split_manifest([record("a")], questions(), fractions=fractions) + self.assertEqual(manifest["assignments"]["a"], "policy") + + def test_independent_representatives_use_neither_gold_values_nor_row_order(self): + rows = [record(str(i), group_id="one-family", language="en" if i % 2 else "de") for i in range(20)] + q = questions() + manifest = make_split_manifest(rows, q) + split = manifest["assignments"]["0"] + chosen = select_independent_records(rows, q, manifest, split=split, question_id="queue") + self.assertEqual(len(chosen), 2) + self.assertEqual({row["language"] for row in chosen}, {"en", "de"}) + self.assertEqual(chosen, select_independent_records(reversed(rows), q, manifest, split=split, question_id="queue")) + for row in rows: + row["targets"]["queue"] = "technical" + changed = select_independent_records(rows, q, make_split_manifest(rows, q), split=split, question_id="queue") + self.assertEqual([row["id"] for row in chosen], [row["id"] for row in changed]) + chosen[0]["source"]["license"] = "changed" + self.assertTrue(all(row["source"]["license"] == "CC0-1.0" for row in rows)) + + def test_missing_labels_empty_partitions_and_invalid_selection(self): + rows = [record("a", group_id="family", targets={"incident": False}), + record("b", group_id="family", targets={"queue": "billing"})] + q, manifest = questions(), make_split_manifest(rows, questions()) + split = manifest["assignments"]["a"] + self.assertEqual([row["id"] for row in select_independent_records(rows, q, manifest, split=split, + question_id="queue")], ["b"]) + empty = next(name for name in SPLITS if name != split) + self.assertEqual(select_independent_records(rows, q, manifest, split=empty, question_id="queue"), []) + self.assertEqual(select_independent_records(rows, q, manifest, split=split, question_id="urgency"), []) + for selected_split, qid in [("unknown", "queue"), (split, "unknown")]: + with self.assertRaises(ValueError): + select_independent_records(rows, q, manifest, split=selected_split, question_id=qid) + + def test_invalid_split_parameters(self): + for update in ({"train": 0}, {"train": -0.1}, {"train": True}, {"train": float("nan")}, + {"train": 0.7}, {"other": 0.1}): + with self.subTest(update=update), self.assertRaises(ValueError): + make_split_manifest([record("a")], questions(), fractions={**DEFAULT_FRACTIONS, **update}) + with self.assertRaises(ValueError): + make_split_manifest([record("a")], questions(), seed="") + + def test_manifest_rejects_tampering(self): + rows, q = [record("a", state={"subject": "invoice", "body": "charged twice"})], questions() + manifest = make_split_manifest(rows, q) + verify_split_manifest(rows, q, manifest) + variants = [] + for field, value in [("schema_version", True), ("schema_version", 2), ("records_sha256", "0" * 64), + ("group_count", True), ("normalized_duplicate_rows", False), ("seed", None)]: + altered = copy.deepcopy(manifest) + altered[field] = value + variants.append(altered) + altered = copy.deepcopy(manifest) + altered["assignments"]["a"] = "unexpected" + variants.append(altered) + for altered in variants: + with self.subTest(manifest=altered), self.assertRaises(ValueError): + verify_split_manifest(rows, q, altered) + rows[0]["state"] = dict(reversed(list(rows[0]["state"].items()))) + with self.assertRaises(ValueError): + verify_split_manifest(rows, q, manifest) + + +class PersistenceTests(unittest.TestCase): + def test_roundtrip_preserves_unicode_and_rendering_order(self): + rows, q = [record("a", state={"z": "café", "a": "Rechnung"})], questions() + q["queue"]["instructions"] = {"z": "first", "a": "second"} + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) / "nested" / "data" + manifest = write_dataset(directory, rows, q) + loaded_q, loaded_rows, loaded_manifest = read_dataset(directory) + self.assertEqual(loaded_q, q) + self.assertEqual(loaded_rows, rows) + self.assertEqual(loaded_manifest, manifest) + self.assertEqual(list(loaded_q["queue"]["criteria"]), ["billing", "technical"]) + self.assertEqual(list(loaded_q["queue"]["instructions"]), ["z", "a"]) + self.assertEqual(list(loaded_rows[0]["state"]), ["z", "a"]) + self.assertNotIn(b"\r\n", (directory / "records.jsonl").read_bytes()) + with self.assertRaises(FileExistsError): + write_dataset(directory, rows, q) + self.assertEqual(read_dataset(directory), (loaded_q, loaded_rows, loaded_manifest)) + + def test_corrupted_disk_data_is_rejected(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) / "data" + write_dataset(directory, [record("a")], questions()) + altered = record("a", targets={"queue": "technical"}) + (directory / "records.jsonl").write_text(json.dumps(altered) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "does not match"): + read_dataset(directory) + + def test_failed_write_does_not_publish_partial_dataset(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) / "data" + with patch.object(Path, "write_text", side_effect=OSError("disk full")): + with self.assertRaisesRegex(OSError, "disk full"): + write_dataset(directory, [record("a")], questions()) + self.assertFalse(directory.exists()) + self.assertEqual(list(Path(temp).iterdir()), []) + + +if __name__ == "__main__": + unittest.main()