Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,12 @@ def __init__(
`fast=True` swaps the encoder/head forward for the TileLang fast path (CUDA only, needs
`pip install laya[fast]`); see `Agent.accelerate`.

`compile=True` runs the model under `torch.compile` and turns ModernBERT's encoder
`reference_compile` on. `torch.compile` specializes per input shape and Laya sees a
new one on almost every request, so those graphs usually cost more than they return;
use it when the traffic is repetitive. `fast=True` takes precedence, because the
TileLang path replaces the forward that would be compiled.

`subfolder` selects one checkpoint from a repo that bundles several, e.g.
`Agent("convaiinnovations/laya", subfolder="multilingual")`. Only that subfolder is
downloaded, so bundling does not cost every user the whole family.
Expand Down Expand Up @@ -1132,6 +1138,7 @@ def __repr__(self) -> str:

def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str] = None,
token: Optional[str] = None, subfolder: Optional[str] = None, fast: bool = False,
compile: bool = False,
revision: Optional[str] = None, expected_sha256: Optional[Dict[str, str]] = None,
lang_temperatures: Optional[Dict[str, Dict[str, Any]]] = None,
hooks=None, on_predict_start=None, on_predict_end=None,
Expand All @@ -1144,12 +1151,14 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
laya.load("convaiinnovations/laya") # English (repo root)
laya.load("convaiinnovations/laya", subfolder="multilingual")
laya.load("convaiinnovations/laya", fast=True) # TileLang GPU fast path
laya.load("convaiinnovations/laya", compile=True) # torch.compile the model

`revision`/`expected_sha256` pin and verify the downloaded artifacts; see `Agent`.
`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see
`laya.hooks`.
"""
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,
compile=compile,
revision=revision, expected_sha256=expected_sha256,
lang_temperatures=lang_temperatures,
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
Expand Down
26 changes: 25 additions & 1 deletion tests/test_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

Run: python tests/test_download.py
"""
import inspect
import json
import os
from pathlib import Path
Expand All @@ -20,9 +21,10 @@
from safetensors.torch import load_file, save_file # noqa: E402
from tokenizers import Tokenizer # noqa: E402
from tokenizers.models import WordLevel # noqa: E402
from torch._dynamo.eval_frame import OptimizedModule # noqa: E402
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402

from laya import load # noqa: E402
from laya import Agent, load # noqa: E402
from laya.common import DecisionModel # noqa: E402


Expand Down Expand Up @@ -138,6 +140,28 @@ def test_loading_skips_initialization_and_preserves_weights(self):
torch.testing.assert_close(value, weights[name], rtol=0, atol=0)
self.assertEqual(agent.predict("hello", self.questions), self.expected)

def test_load_signature_names_every_agent_option(self):
# `load` is a pass-through, so an `Agent` option it does not name is unreachable
# from the entry point the guides recommend. Read off the signatures rather than
# written down, so the next option added to Agent has to be forwarded here too.
unreachable = (set(inspect.signature(Agent.__init__).parameters) - {"self"}
- set(inspect.signature(load).parameters))
self.assertEqual(unreachable, set(),
"laya.load cannot set: %s" % ", ".join(sorted(unreachable)))

def test_compile_true_wraps_the_model_loaded_by_load(self):
# The observable `Agent.__init__` produces for compile=True, reached through load().
# Nothing is actually compiled here: torch.compile wraps eagerly and specializes on
# the first forward, so this stays a weight-free-of-cost check.
eager = load(str(self.repo), device="cpu")
self.assertFalse(eager.model.encoder.config.reference_compile)
self.assertNotIsInstance(eager.model, OptimizedModule)

compiled = load(str(self.repo), device="cpu", compile=True)
self.assertTrue(compiled.model.encoder.config.reference_compile)
self.assertIsInstance(compiled.model, OptimizedModule)
self.assertIsInstance(compiled.model._orig_mod, DecisionModel)


if __name__ == "__main__":
torch.set_num_threads(1)
Expand Down
4 changes: 4 additions & 0 deletions tests/test_hooks_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ def check_param(name, fn, param, default, kind=None):
check_param("Router.__init__", Router.__init__, "lang_guess", None)
check_param("Router.__init__", Router.__init__, "revisions", None)

# load() forwards Agent's own construction options, so none of them is reachable only
# through the class; tests/test_download.py asserts that against both signatures.
check_param("load", load, "compile", False)

# --------------------------------------------------------------- predict surfaces
for label, fn in (("Agent.predict_batch", Agent.predict_batch),
("Agent.system_one", Agent.system_one),
Expand Down
Loading