Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,12 @@ router.unload() # free memory
| `Router(max_loaded=1)` | 7 to 10 s on every language switch | 1 per switch |
| `Router(preload=True)` | **32.8 ms (GPU) / 193–464 ms (CPU)** | **none** |

A rebuild still re-reads the checkpoint, but each checkpoint's tokenizer is parsed once per process
and reused by every `Agent` — including one the Router rebuilds after eviction. The multilingual
`tokenizer.json` alone is 34 MB / 256k vocab, several times the cost of applying its weights.
Preloading is still the right answer for a server: it removes the rebuild rather than making it
cheaper.

### Supplying Your Own Language Detection

Routing asks one question: *can the English checkpoint read this state?* The built-in detector answers it from the script and a function-word heuristic, and is deliberately dependency-free. That heuristic is best-effort on Latin-script languages it holds no word list for, so a short request can carry no usable signal:
Expand Down
40 changes: 38 additions & 2 deletions laya/agent.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""High-level inference runtime for laya System 1 decision models."""
import json
import os
import threading
import warnings
from typing import Any, Dict, Optional, Union

Expand Down Expand Up @@ -97,6 +98,42 @@ def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str,
)


_TOKENIZERS: Dict[tuple, Any] = {}
_TOKENIZERS_LOCK = threading.Lock()


def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:
"""Tokenizer for a checkpoint, parsed once per process.

Parsing `tokenizer.json` is not free and `huggingface_hub` caches only the download, not
the parsed object: the multilingual checkpoint ships a 34 MB, 256k-vocabulary file that
costs seconds to load, several times the cost of applying its weights. A tokenizer is
read-only during inference, so one instance is shared by every Agent that wants the same
directory -- including an Agent the Router has rebuilt after eviction.

Keyed on the tokenizer directory and its mtime, so a re-download or a config rewritten by
`_fix_tokenizer_config()` still produces a fresh parse. Non-directory sources (a hub id)
are not cached, so a caller cannot pin a stale remote revision.
"""
from transformers import AutoTokenizer

if not os.path.isdir(tok_dir):
return AutoTokenizer.from_pretrained(cfg.get("encoder"))

try:
stamp = (os.path.abspath(tok_dir), os.path.getmtime(os.path.join(tok_dir, "tokenizer_config.json")))
except OSError:
return AutoTokenizer.from_pretrained(tok_dir)

with _TOKENIZERS_LOCK:
cached = _TOKENIZERS.get(stamp)
if cached is not None:
return cached
tokenizer = AutoTokenizer.from_pretrained(tok_dir)
_TOKENIZERS[stamp] = tokenizer
return tokenizer


class Agent:
"""System 1 decision model runtime: fast, non-autoregressive, calibrated decisions."""

Expand All @@ -114,7 +151,6 @@ def __init__(
downloaded, so bundling does not cost every user the whole family.
"""
from safetensors.torch import load_file
from transformers import AutoTokenizer
try:
from transformers.initialization import no_init_weights
except ImportError: # Transformers 4.x
Expand Down Expand Up @@ -186,7 +222,7 @@ def __init__(
self.device = torch.device("cpu")

tok_dir = os.path.join(model_dir, "tokenizer")
self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))
self.tok = _load_tokenizer(tok_dir, self.cfg)

enc_dir = os.path.join(model_dir, "encoder")
# The checkpoint supplies every parameter; skip random/base-model weights.
Expand Down
19 changes: 19 additions & 0 deletions tests/test_local_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,25 @@ def head(t):
ok("explicit typed-decisions honoured", res_td["routing"]["model"] == "typed-decisions")
ok("routing payload serialises", isinstance(json.dumps(res_td["routing"]), str))

# ---------------------------------------------------------------- 6. tokenizer reuse
head("6. Tokenizers are parsed once per checkpoint, not per Agent")
# `huggingface_hub` caches the download but not the parsed tokenizer, and the eviction above
# destroyed the whole Agent. Rebuilding it must not re-parse tokenizer.json -- 34 MB on the
# multilingual checkpoint, several times the cost of applying its weights.
first = laya.load(LOCAL["english"], device=DEVICE)
again = laya.load(LOCAL["english"], device=DEVICE)
ok("same checkpoint reuses its tokenizer", first.tok is again.tok)

multi = laya.load(LOCAL["multilingual"], device=DEVICE)
ok("a different checkpoint gets its own tokenizer", multi.tok is not first.tok)

tok_dir = os.path.join(LOCAL["english"], "tokenizer")
os.utime(os.path.join(tok_dir, "tokenizer_config.json"), None)
refreshed = laya.load(LOCAL["english"], device=DEVICE)
ok("a rewritten tokenizer config forces a fresh parse", refreshed.tok is not first.tok,
"an edited on-disk tokenizer must not be masked by the cache")
del first, again, multi, refreshed

# ---------------------------------------------------------------- summary
head("SUMMARY")
for n in NOTES:
Expand Down