Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1824,9 +1824,27 @@ def save_calibration(self, path: str) -> None:
subfolder=getattr(self, "subfolder", None),
config=getattr(self, "cfg", None),
)
with open(path, "w") as f:
json.dump(payload, f, indent=2)
f.write("\n")
destination = os.path.realpath(path)
fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination), prefix=".calibration.", suffix=".tmp")
try:
with os.fdopen(fd, "w") as f:
json.dump(payload, f, indent=2)
f.write("\n")
f.flush()
os.fsync(f.fileno())
try:
mode = os.stat(destination).st_mode & 0o777
except FileNotFoundError:
pass
else:
os.chmod(temporary, mode)
os.replace(temporary, destination)
except BaseException:
try:
os.unlink(temporary)
except OSError:
pass
raise

def load_calibration(self, path: str) -> None:
"""Read a JSON map written by `save_calibration` onto this agent.
Expand Down
63 changes: 51 additions & 12 deletions laya/calibrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
bounds in this module.
"""
import warnings
from numbers import Integral
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple

import numpy as np
Expand Down Expand Up @@ -55,14 +56,43 @@ def _vec(x) -> np.ndarray:
return np.asarray(x, dtype=np.float32).reshape(-1)


def _validated_pair(logits, target, k=None):
vectors = []
for name, value in (("logits", logits), ("target", target)):
if isinstance(value, torch.Tensor):
value = value.detach().cpu().numpy()
try:
vector = np.asarray(value)
if vector.ndim != 1 or vector.dtype.kind not in "iuf":
raise ValueError("%s must be a one-dimensional numeric vector" % name)
with np.errstate(over="ignore", invalid="ignore"):
vector = vector.astype(np.float32)
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError("%s must be a one-dimensional numeric vector" % name) from exc
if not np.isfinite(vector).all():
raise ValueError("%s must contain only finite values" % name)
vectors.append(vector)
logits, target = vectors
if len(logits) != len(target):
raise ValueError("logits and target must have the same length")
if k is None:
k = len(logits)
if isinstance(k, bool) or not isinstance(k, Integral) or k < 1 or k > len(logits):
raise ValueError("k must be an integer between 1 and the vector length")
logits, target = logits[:k], target[:k]
if (target < 0).any() or not np.isclose(target.sum(), 1.0, rtol=1e-5, atol=1e-6):
raise ValueError("target must be a nonnegative probability vector summing to 1")
return logits, target, int(k)


def _pairs_to_tensors(pairs: Sequence) -> Tuple[torch.Tensor, torch.Tensor]:
kmax = max(len(_vec(z)) for z, _ in pairs)
z_mat = torch.full((len(pairs), kmax), -1e4)
t_mat = torch.zeros((len(pairs), kmax))
for i, (z, t) in enumerate(pairs):
z = _vec(z)
t = _vec(t)
n = min(len(z), len(t))
n = len(z)
z_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(z[:n]))
t_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(t[:n]))
return z_mat, t_mat
Expand All @@ -80,7 +110,16 @@ def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
"""
if min_n is None:
min_n = MIN_BUCKET_N
sel = list(pairs)
if isinstance(min_n, bool) or not isinstance(min_n, Integral) or min_n < 1:
raise ValueError("min_n must be a positive integer")
sel = []
for pair in pairs:
try:
z, t = pair
except (TypeError, ValueError) as exc:
raise ValueError("pair must be (logits, target)") from exc
z, t, _k = _validated_pair(z, t)
sel.append((z, t))
if len(sel) < min_n:
return 1.0
z_mat, t_mat = _pairs_to_tensors(sel)
Expand All @@ -102,19 +141,19 @@ def closure():
def _iter_records(records: Iterable) -> List[Tuple[int, np.ndarray, np.ndarray, int]]:
out = []
for rec in records:
if not isinstance(rec, (list, tuple)) or len(rec) not in (3, 4):
raise ValueError("record must be (qtype, logits, target[, k])")
if len(rec) == 4:
qtype, logits, target, k = rec
elif len(rec) == 3:
qtype, logits, target = rec
k = len(_vec(logits))
if k is None:
raise ValueError("k must be an integer between 1 and the vector length")
else:
raise ValueError("record must be (qtype, logits, target[, k])")
logits = _vec(logits)
target = _vec(target)
k = int(k)
if k < 1:
raise ValueError("k must be >= 1")
out.append((int(qtype), logits[:k], target[:k], k))
qtype, logits, target = rec
k = None
if isinstance(qtype, bool) or not isinstance(qtype, Integral) or not 0 <= qtype < N_QTYPES:
raise ValueError("qtype must be an integer between 0 and %d" % (N_QTYPES - 1))
logits, target, k = _validated_pair(logits, target, k)
out.append((int(qtype), logits, target, k))
return out


Expand Down
148 changes: 148 additions & 0 deletions tests/test_calibrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@
import sys
import tempfile
import warnings
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from threading import Barrier
from unittest.mock import patch

import numpy as np

Expand All @@ -33,6 +37,7 @@
)
from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402
import laya.calibrate as _calibrate # noqa: E402
import laya.agent as _agent_module # noqa: E402

PASS, FAIL = [], []

Expand Down Expand Up @@ -211,6 +216,59 @@ def overconfident_records(n, k, qtype, seed=0, acc=0.6, mag=8.0):


# --------------------------------------------------------------- live Agent map + JSON round-trip

def _refuses_records(name, records, fragment):
try:
fit_temperature_map(records)
except ValueError as exc:
check_true("records/" + name, fragment in str(exc), str(exc))
except Exception as exc:
FAIL.append("records/%s: raised %r, want ValueError" % (name, exc))
else:
FAIL.append("records/%s: invalid input was accepted" % name)


for name, record, field in (
("scalar record", 3, "record"),
("wrong record length", (0, [1, 2]), "record"),
("bool type", (True, [1, 2], [1, 0]), "qtype"),
("fractional type", (0.9, [1, 2], [1, 0]), "qtype"),
("unknown type", (3, [1, 2], [1, 0]), "qtype"),
("negative type", (-1, [1, 2], [1, 0]), "qtype"),
("bool width", (0, [1, 2], [1, 0], True), "k"),
("null width", (0, [1, 2], [1, 0], None), "k"),
("fractional width", (0, [1, 2], [1, 0], 1.9), "k"),
("zero width", (0, [1, 2], [1, 0], 0), "k"),
("oversized width", (0, [1, 2], [1, 0], 3), "k"),
("empty vectors", (0, [], []), "k"),
("matrix logits", (0, [[1, 2]], [1, 0]), "logits"),
("matrix target", (0, [1, 2], [[1, 0]]), "target"),
("text logits", (0, ["1", "2"], [1, 0]), "logits"),
("nan logits", (0, [np.nan, 2], [1, 0]), "logits"),
("infinite target", (0, [1, 2], [np.inf, 0]), "target"),
("unequal vectors", (0, [1, 2], [1]), "same length"),
("negative target", (0, [1, 2], [-0.1, 1.1]), "probability"),
("unnormalized target", (0, [1, 2], [1, 1]), "probability"),
):
_refuses_records(name, [record], field)

soft_record = _iter_records([(np.int64(0), [1, 2, 0], [0.3, 0.7, 0], np.int64(2))])[0]
check("records/numpy integer and explicit padding", soft_record[3], 2)
check_true("records/soft target kept", np.allclose(soft_record[2], [0.3, 0.7]))
check_true("records/soft target fits", np.isfinite(fit_one_temperature([(soft_record[1], soft_record[2])], min_n=1)))
check("records/empty dataset still neutral", fit_temperature_map([])["temperature"], [1.0] * 3)
for name, pairs, min_n in (
("mismatched lengths", [([1, 2], [1])], None),
("invalid below sample floor", [([np.nan, 2], [1, 0])], None),
("invalid min_n", [], 0),
):
try:
fit_one_temperature(pairs, min_n=min_n)
except ValueError:
PASS.append("pairs/" + name)
else:
FAIL.append("pairs/%s: invalid input was accepted" % name)

_CFG = {
"encoder": "answerdotai/ModernBERT-large",
"head_layers": 2,
Expand Down Expand Up @@ -256,6 +314,96 @@ def overconfident_records(n, k, qtype, seed=0, acc=0.6, mag=8.0):
)
check_true("save/no weights key", "model.safetensors" not in json.dumps(payload))


# --------------------------------------------------------------- failed saves preserve the old map
with tempfile.TemporaryDirectory() as atomic_dir:
atomic_path = Path(atomic_dir) / "calibration.json"
agent.save_calibration(atomic_path)
original_bytes = atomic_path.read_bytes()
original_replace = os.replace

def _partial_dump(body, stream, **kwargs):
stream.write('{"temperature": [')
raise OSError("injected write failure")

for failure, context in (
("write", patch.object(_agent_module.json, "dump", _partial_dump)),
("sync", patch.object(_agent_module.os, "fsync", side_effect=OSError("injected sync failure"))),
("replace", patch.object(_agent_module.os, "replace", side_effect=OSError("injected replace failure"))),
):
with context:
try:
agent.save_calibration(atomic_path)
except OSError as exc:
check_true("atomic/%s error propagates" % failure, "injected" in str(exc))
else:
FAIL.append("atomic/%s error did not propagate" % failure)
check("atomic/%s keeps prior bytes" % failure, atomic_path.read_bytes(), original_bytes)
check("atomic/%s cleans temp" % failure, sorted(os.listdir(atomic_dir)), ["calibration.json"])

missing_path = Path(atomic_dir) / "new.json"
with patch.object(_agent_module.json, "dump", _partial_dump):
try:
agent.save_calibration(missing_path)
except OSError:
pass
else:
FAIL.append("atomic/first-save error did not propagate")
check_true("atomic/failed first save leaves no destination", not missing_path.exists())
check("atomic/failed first save cleans temp", sorted(os.listdir(atomic_dir)), ["calibration.json"])

# Successful updates keep the bytes/format and the existing readers' permissions.
agent.save_calibration(atomic_path)
check("atomic/success preserves JSON format", atomic_path.read_bytes(), original_bytes)
if os.name != "nt":
os.chmod(atomic_path, 0o640)
agent.save_calibration(atomic_path)
check("atomic/preserves mode", atomic_path.stat().st_mode & 0o777, 0o640)
link_path = Path(atomic_dir) / "linked.json"
link_path.symlink_to(atomic_path.name)
agent.save_calibration(link_path)
check_true("atomic/keeps symlink", link_path.is_symlink())
check("atomic/writes symlink target", atomic_path.read_bytes(), original_bytes)
link_path.unlink()

rival = Agent.__new__(Agent)
rival.temperature = [1.1, 1.2, 1.3]
rival.temperature_by_options = {"choice:2": 1.4}
for simulate_lost_race in (False, True):
barrier = Barrier(2)

def _together_replace(src, dst):
rank = barrier.wait(timeout=10)
if simulate_lost_race and rank == 0:
error = PermissionError("injected NTFS replacement race")
error.winerror = 5
raise error
original_replace(src, dst)

succeeded = 0
denied = 0
with patch.object(_agent_module.os, "replace", _together_replace):
with ThreadPoolExecutor(max_workers=2) as pool:
writes = [pool.submit(writer.save_calibration, atomic_path) for writer in (agent, rival)]
for write in writes:
try:
write.result()
except PermissionError as exc:
# NTFS may deny one simultaneous replacement; other errors still fail.
if getattr(exc, "winerror", None) != 5:
raise
denied += 1
else:
succeeded += 1
label = "injected race" if simulate_lost_race else "concurrent writers"
check_true("atomic/%s has a successful writer" % label, succeeded >= 1)
if simulate_lost_race:
check("atomic/injected race exercises denial", denied, 1)
concurrent_payload = json.loads(atomic_path.read_text())
check_true("atomic/%s leave one complete payload" % label,
concurrent_payload in (payload, calibration_payload(rival.temperature, rival.temperature_by_options)))
check("atomic/%s clean temps" % label, sorted(os.listdir(atomic_dir)), ["calibration.json"])

other = Agent.__new__(Agent)
other.model_id_or_path = agent.model_id_or_path
other.subfolder = agent.subfolder
Expand Down