Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ def __init__(
"""
from safetensors.torch import load_file
from transformers import AutoTokenizer
try:
from transformers.initialization import no_init_weights
except ImportError: # Transformers 4.x
from transformers.modeling_utils import no_init_weights

model_dir = model_id_or_path
if not os.path.exists(model_dir):
Expand Down Expand Up @@ -185,7 +189,10 @@ def __init__(
self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))

enc_dir = os.path.join(model_dir, "encoder")
self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None)
# The checkpoint supplies every parameter; skip random/base-model weights.
with no_init_weights():
self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None,
pretrained=False)

# Load weights and verify architectural compatibility
weights = load_file(weights_path)
Expand Down
6 changes: 3 additions & 3 deletions laya/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,11 @@ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, det
return logits, act_logits


def build_model(cfg: Dict, encoder_dir: Optional[str] = None) -> DecisionModel:
def build_model(cfg: Dict, encoder_dir: Optional[str] = None, pretrained: bool = True) -> DecisionModel:
from transformers import AutoConfig, AutoModel

if encoder_dir and os.path.exists(encoder_dir):
ecfg = AutoConfig.from_pretrained(encoder_dir)
if not pretrained or (encoder_dir and os.path.exists(encoder_dir)):
ecfg = AutoConfig.from_pretrained(encoder_dir or cfg["encoder"])
enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
else:
enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")
Expand Down
25 changes: 24 additions & 1 deletion tests/test_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import torch # noqa: E402
from huggingface_hub.utils import filter_repo_objects # noqa: E402
from safetensors.torch import save_file # noqa: E402
from safetensors.torch import load_file, save_file # noqa: E402
from tokenizers import Tokenizer # noqa: E402
from tokenizers.models import WordLevel # noqa: E402
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
Expand Down Expand Up @@ -113,6 +113,29 @@ def test_local_paths_do_not_download(self):
self.assertEqual(agent.predict("hello", self.questions), self.expected)
download.assert_not_called()

def test_loading_skips_initialization_and_preserves_weights(self):
weights = load_file(self.repo / "model.safetensors")
for bundled_encoder in (True, False):
with self.subTest(bundled_encoder=bundled_encoder), tempfile.TemporaryDirectory() as destination:
path = Path(destination)
for filename in self.runtime_files:
if not bundled_encoder and filename.startswith("encoder/"):
continue
target = path / filename
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(self.repo / filename, target)
cfg_path = path / "rl_agent_config.json"
cfg = json.loads(cfg_path.read_text())
cfg["encoder"] = str(self.repo / "encoder")
cfg_path.write_text(json.dumps(cfg))
rng = torch.get_rng_state()
with patch("transformers.AutoModel.from_pretrained", side_effect=AssertionError("Unused base weights")):
agent = load(str(path), device="cpu")
self.assertTrue(torch.equal(torch.get_rng_state(), rng), "Loading initialized random weights")
for name, value in agent.model.state_dict().items():
torch.testing.assert_close(value, weights[name], rtol=0, atol=0)
self.assertEqual(agent.predict("hello", self.questions), self.expected)


if __name__ == "__main__":
torch.set_num_threads(1)
Expand Down