Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
perf: skip random parameter init on checkpoint load
A cold laya.load() spent most of its time on work that is thrown away: the
model was constructed with AutoModel.from_config(), which random-initialises
every parameter, and load_state_dict() then overwrote all of them.

On a 3-thread CPU box the ~19 s cold load broke down as ~17 s of construction
against ~1.1 s to read 0.84 GB of safetensors and ~0.3 s to apply them.
Pinning the parameters to the meta device during construction removes the
wasted initialisation while leaving every buffer intact.

Buffers must NOT be skipped: ModernBERT computes its non-persistent RoPE
inv_freq buffers in __init__, they are absent from every checkpoint, and
forward() reads them. Materialising with nn.Module.to_empty() zeroes them,
which leaves the encoder silently attending with a dead rotary embedding, so
materialise_meta_parameters() converts parameters only.
  • Loading branch information
wangtiantian
wangtiantian committed Sep 22, 2026
commit ee1296da1eec07da87434a37766d96d4bf153f9b
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ The English checkpoint collapses on non-Latin scripts (Khmer scores **0.000 accu

### Production Preload & Memory

A cold checkpoint build costs seconds; language detection costs microseconds. At the default `max_loaded=1`, traffic that alternates languages rebuilds a model on *every* request (measured at a 7.4 s median reload on CPU and 10.3 s on T4).
A cold checkpoint build costs seconds; language detection costs microseconds. At the default `max_loaded=1`, traffic that alternates languages rebuilds a model on *every* request (measured at a 7.4 s median reload on CPU and 10.3 s on T4). Only a fraction of that is the weights: applying 0.84 GB of them takes ~0.4 s once the file is read, and the rest was random initialisation of ~400M parameters that the checkpoint load immediately overwrites.

For a server or production app, preload:

Expand Down
11 changes: 8 additions & 3 deletions laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
clamp_temperature,
collate_items,
confidence_from_probs,
materialise_meta_parameters,
render_options,
temp_bucket,
)
Expand Down Expand Up @@ -185,12 +186,16 @@ def __init__(
self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))

enc_dir = os.path.join(model_dir, "encoder")
self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None)

# Load weights and verify architectural compatibility
# Load weights and verify architectural compatibility. The model is built with
# uninitialised `meta` parameters because `load_state_dict` below supplies every one of
# them; letting nn.Module.__init__ randomise ~400M parameters first roughly triples the
# cost of a cold load and the result is discarded immediately.
weights = load_file(weights_path)
self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None,
empty=True)
_verify_compatibility(self.model, self.cfg, weights, model_id_or_path)

materialise_meta_parameters(self.model, device="cpu")
self.model.load_state_dict(weights, strict=True)

# ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder.
Expand Down
71 changes: 64 additions & 7 deletions laya/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import json
import math
import os
from contextlib import contextmanager, nullcontext
from typing import Dict, List, Optional, Union

import numpy as np
Expand Down Expand Up @@ -136,15 +137,71 @@ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, det
return logits, act_logits


def build_model(cfg: Dict, encoder_dir: Optional[str] = None) -> DecisionModel:
@contextmanager
def _meta_parameters():
"""Create parameters on the `meta` device while everything else builds normally.

Training-free inference loads every parameter from a checkpoint, so the random init that
`nn.Module.__init__` performs is pure waste -- and it is not cheap. On a 3-thread CPU box it
accounts for ~17 s of the ~19 s a cold `laya.load()` takes, against ~1.4 s to read 0.84 GB of
weights and apply them. Pinning parameters to `meta` skips that allocation and initialisation.

Buffers are deliberately left alone. They must still be initialised: ModernBERT computes its
non-persistent RoPE `inv_freq` buffers in `__init__`, they are absent from the checkpoint, and
they are read during the forward pass. `to_empty()` would materialise them as zeros and the
encoder would then silently attend with a dead rotary embedding.

Only `register_parameter` is shadowed, so nested modules and `None` parameters behave exactly
as before. Callers must materialise the parameters with `materialise_meta_parameters()` before
the module is used.
"""
original = nn.Module.register_parameter

def register_parameter(module, name, param):
if param is None:
module._parameters[name] = None
return
module._parameters[name] = nn.Parameter(
torch.empty(param.shape, dtype=param.dtype, device="meta"),
requires_grad=param.requires_grad)

nn.Module.register_parameter = register_parameter
try:
yield
finally:
nn.Module.register_parameter = original


def materialise_meta_parameters(module: nn.Module, device="cpu") -> nn.Module:
"""Give every `meta` parameter real storage on `device`, in place.

`to_empty()` would also re-create the buffers, discarding the values `_meta_parameters()`
carefully preserved, so only parameters are converted here.
"""
for submodule in module.modules():
for name, param in submodule._parameters.items():
if param is not None and param.device.type == "meta":
submodule._parameters[name] = nn.Parameter(
torch.empty(param.shape, dtype=param.dtype, device=device),
requires_grad=param.requires_grad)
return module


def build_model(cfg: Dict, encoder_dir: Optional[str] = None, empty: bool = False) -> DecisionModel:
"""Build the architecture.

`empty=True` leaves the parameters uninitialised on the `meta` device, which is what a
checkpoint load wants. The caller must call `materialise_meta_parameters()` before use.
"""
from transformers import AutoConfig, AutoModel

if encoder_dir and os.path.exists(encoder_dir):
ecfg = AutoConfig.from_pretrained(encoder_dir)
enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
else:
enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")
return DecisionModel(enc, cfg.get("head_layers", 2), len(cfg.get("act_costs", {})) + 1)
with _meta_parameters() if empty else nullcontext():
if encoder_dir and os.path.exists(encoder_dir):
ecfg = AutoConfig.from_pretrained(encoder_dir)
enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
else:
enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")
return DecisionModel(enc, cfg.get("head_layers", 2), len(cfg.get("act_costs", {})) + 1)


def proper_reward(
Expand Down
24 changes: 24 additions & 0 deletions research/scripts/bench_load_latency.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
"""Cold-load cost of a Laya checkpoint, measured as `Agent()` constructions after a warmup.

This is the number the README's "Production Preload & Memory" section talks about: with the
Router's default `max_loaded=1`, alternating languages rebuilds an Agent on every request.

Run: python3 research/scripts/bench_load_latency.py
"""
import os, sys, time, gc, statistics, shutil
os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", USE_TF="0", USE_TORCH="1",
TOKENIZERS_PARALLELISM="false")
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya.agent import Agent
snap = os.path.expanduser("~/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/1c5edc17a7acd8701df6fc341c0d179f1c62c982")
for label, sub in (("english",""),("multilingual","multilingual"),("typed-decisions","typed-decisions")):
work = "/tmp/bench_%s" % (sub or "english")
shutil.rmtree(work, ignore_errors=True)
shutil.copytree(os.path.join(snap, sub) if sub else snap, work)
Agent(work, device="cpu") # warm: imports, module registration, page cache
ts = []
for _ in range(3):
gc.collect(); t = time.perf_counter(); ag = Agent(work, device="cpu")
ts.append(time.perf_counter() - t); del ag; gc.collect()
print(" %-18s median %6.2f s (%s)" % (label, statistics.median(ts),
", ".join("%.2f" % x for x in ts)))
103 changes: 102 additions & 1 deletion tests/test_decision_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@
"""
import os
import sys
from contextlib import nullcontext

import torch
from transformers import AutoConfig, AutoModel

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from laya.common import DecisionModel
from laya.common import DecisionModel, _meta_parameters, materialise_meta_parameters


def _tiny_model(head_layers: int = 1, n_act: int = 2) -> DecisionModel:
Expand Down Expand Up @@ -93,9 +95,108 @@ def test_multi_option_question_is_unaffected():
assert torch.isfinite(act_logits).all()


def _tiny_rope_model(empty: bool = False) -> DecisionModel:
"""A tiny ModernBERT-backed DecisionModel, the point being its RoPE buffers.

ModernBERT computes non-persistent rotary `inv_freq` buffers in __init__, they are absent
from every checkpoint, and forward() reads them. They are the reason a model cannot simply
be constructed empty and materialised with to_empty().

Built from a config rather than `build_model`, so no pretrained weights are fetched.
"""
cfg = AutoConfig.for_model(
"modernbert",
hidden_size=32,
num_hidden_layers=2,
num_attention_heads=2,
intermediate_size=64,
vocab_size=100,
max_position_embeddings=64,
pad_token_id=0,
eos_token_id=1,
bos_token_id=2,
cls_token_id=2,
sep_token_id=1,
)
with _meta_parameters() if empty else nullcontext():
encoder = AutoModel.from_config(cfg, attn_implementation="sdpa")
return DecisionModel(encoder, head_layers=1, n_act=1).eval()


def test_empty_build_defers_parameters_but_keeps_buffers():
# Regression test for the cold-load path: parameters are created on `meta` so the random
# init that a checkpoint load immediately overwrites is never paid, while buffers must
# still be initialised because they are computed, not loaded.
import torch.nn as nn

original = nn.Module.register_parameter
reference = _tiny_rope_model(empty=False)
assert nn.Module.register_parameter is original, "register_parameter was not restored"

empty = _tiny_rope_model(empty=True)
params = list(empty.parameters())
assert params, "the model should still expose its parameters"
assert all(p.device.type == "meta" for p in params), "parameters should live on meta"
assert nn.Module.register_parameter is original, "register_parameter leaked out of the build"

# Buffers are not deferred, and the RoPE ones hold what __init__ computed rather than zeros.
empty_rope = {n: b for n, b in empty.named_buffers() if "inv_freq" in n}
reference_rope = {n: b for n, b in reference.named_buffers() if "inv_freq" in n}
assert empty_rope, "ModernBERT should expose rotary inv_freq buffers"
assert set(empty_rope) == set(reference_rope)
for name, buf in empty_rope.items():
assert buf.device.type != "meta", "%s should be a real buffer" % name
assert buf.abs().sum() > 0, "%s is zeroed; RoPE would be dead" % name
assert torch.equal(buf, reference_rope[name]), "%s differs from a normal build" % name


def test_materialising_does_not_disturb_the_rope_buffers():
# The trap this pins down: nn.Module.to_empty() re-creates *buffers* as well as parameters,
# and the rotary inv_freq buffers are computed in __init__ and absent from every checkpoint.
# Materialising that way leaves them zeroed, so the encoder silently attends with a dead
# rotary embedding. Only the parameters may be touched.
empty = _tiny_rope_model(empty=True)
before = {n: b.clone() for n, b in empty.named_buffers()}
assert before, "the model should expose buffers"

materialise_meta_parameters(empty, device="cpu")

after = dict(empty.named_buffers())
for name, original in before.items():
assert original.abs().sum() > 0, "%s should have been computed at build time" % name
assert torch.equal(after[name], original), "%s was disturbed by materialisation" % name


def test_materialised_empty_model_matches_a_normal_build():
# The end-to-end guarantee for the fast cold-load path: building empty, materialising the
# parameters and loading a state dict must reproduce a normal build exactly.
torch.manual_seed(3)
reference = _tiny_rope_model(empty=False)
input_ids, attention_mask, marker_pos, marker_mask, qtype = _inputs(
batch=2, seq=8, n_markers=3
)
with torch.no_grad():
expected_logits, expected_act = reference(
input_ids, attention_mask, marker_pos, marker_mask, qtype
)

loaded = _tiny_rope_model(empty=True)
materialise_meta_parameters(loaded, device="cpu")
assert all(p.device.type == "cpu" for p in loaded.parameters())
loaded.load_state_dict(reference.state_dict(), strict=True)

with torch.no_grad():
logits, act_logits = loaded(input_ids, attention_mask, marker_pos, marker_mask, qtype)
assert torch.equal(logits, expected_logits), "empty-then-loaded logits diverged"
assert torch.equal(act_logits, expected_act), "empty-then-loaded act logits diverged"


if __name__ == "__main__":
test_single_option_question_does_not_crash()
test_single_option_top1_minus_top2_is_exactly_one()
test_multi_option_question_is_unaffected()
test_empty_build_defers_parameters_but_keeps_buffers()
test_materialising_does_not_disturb_the_rope_buffers()
test_materialised_empty_model_matches_a_normal_build()
print("all decision model tests passed")