Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/docker.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ These apply to the `laya-serve` service only.
| `LAYA_AUTO_TASK` | `0` | `1` lets the router reach `typed-decisions` automatically |
| `LAYA_LOG_LEVEL` | `info` | uvicorn log level |
| `LAYA_API_KEY` | (none) | when set, requires `Authorization: Bearer <key>` |
| `LAYA_MAX_TOKEN_BUDGET` | `8192` | cap on per-request `max_len` and `head_max_len` overrides |

`LAYA_PRELOAD` defaults to `0` here rather than the package default of `1`, because
preloading makes the first boot download all three checkpoints. Set it to `1` for a
Expand Down
92 changes: 72 additions & 20 deletions laya/serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,24 +11,25 @@
Configuration is entirely via environment variables so the same entry point
serves a laptop dev run and a systemd unit:

====================== ============================================ =========
env var meaning default
====================== ============================================ =========
``LAYA_HOST`` bind address 0.0.0.0
``LAYA_PORT`` bind port 8000
``LAYA_DEVICE`` torch device for every checkpoint (auto)
``LAYA_PRELOAD`` build the checkpoints at startup, not lazily 1
``LAYA_MODELS`` comma list to preload (english,multilingual, (all)
typed-decisions); empty = every checkpoint
``LAYA_THREADS`` cap torch intra-op threads (CPU inference). (torch
Keep <= physical cores; oversubscribing the default)
logical/hyperthread count is a large regression.
``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0
``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)
``LAYA_LOG_LEVEL`` uvicorn log level info
``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16
gets 503 (see below)
====================== ============================================ =========
========================= ============================================ =========
env var meaning default
========================= ============================================ =========
``LAYA_HOST`` bind address 0.0.0.0
``LAYA_PORT`` bind port 8000
``LAYA_DEVICE`` torch device for every checkpoint (auto)
``LAYA_PRELOAD`` build the checkpoints at startup, not lazily 1
``LAYA_MODELS`` comma list to preload (english,multilingual, (all)
typed-decisions); empty = every checkpoint
``LAYA_THREADS`` cap torch intra-op threads (CPU inference). (torch
Keep <= physical cores; oversubscribing the default)
logical/hyperthread count is a large regression.
``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0
``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)
``LAYA_LOG_LEVEL`` uvicorn log level info
``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16
gets 503 (see below)
``LAYA_MAX_TOKEN_BUDGET`` cap on per-request max_len / head_max_len 8192
========================= ============================================ =========

Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are all
deferred into the functions that need them, so ``import laya.serve`` stays cheap
Expand Down Expand Up @@ -68,6 +69,8 @@
# before inference, so without a bound many concurrent near-cap requests OOM
# the worker even though every request is individually valid (#330).
DEFAULT_MAX_CONCURRENT = 16
# Server-side ceiling on per-request max_len/head_max_len token budget overrides.
DEFAULT_MAX_TOKEN_BUDGET = 8192
# Public Hugging Face ids, accepted so a client can name a checkpoint. The root bundle is
# deliberately absent: the documented ``convaiinnovations/laya`` value means
# "let the Router choose", rather than pinning the English checkpoint.
Expand Down Expand Up @@ -115,6 +118,43 @@ def _resolve_max_concurrent() -> int:
return n if n > 0 else DEFAULT_MAX_CONCURRENT


def _resolve_max_token_budget() -> int:
"""Server-side cap on per-request max_len from LAYA_MAX_TOKEN_BUDGET."""
raw = os.environ.get("LAYA_MAX_TOKEN_BUDGET")
if not raw:
return DEFAULT_MAX_TOKEN_BUDGET
try:
n = int(str(raw).strip())
except (TypeError, ValueError):
_log.warning("invalid LAYA_MAX_TOKEN_BUDGET %r; falling back to %d", raw, DEFAULT_MAX_TOKEN_BUDGET)
return DEFAULT_MAX_TOKEN_BUDGET
if n <= 0:
_log.warning("LAYA_MAX_TOKEN_BUDGET must be positive (got %d); falling back to %d", n, DEFAULT_MAX_TOKEN_BUDGET)
return DEFAULT_MAX_TOKEN_BUDGET
return n


def _validate_budget_param(body: Dict[str, Any], key: str, max_cap: int) -> Optional[int]:
"""Validate optional integer budget parameters (max_len / head_max_len) from request body (422)."""
from fastapi import HTTPException

if key not in body:
return None
val = body[key]
if val is None:
return None
if not isinstance(val, int) or isinstance(val, bool):
raise HTTPException(status_code=422, detail="%s must be an integer" % key)
if val <= 0:
raise HTTPException(status_code=422, detail="%s must be a positive integer" % key)
if val > max_cap:
raise HTTPException(
status_code=422,
detail="%s exceeds server limit (%d > %d)" % (key, val, max_cap),
)
return val


def _resolve_port() -> int:
"""Port from LAYA_PORT, validated. Exits with a message instead of a traceback."""
raw = os.environ.get("LAYA_PORT", "8000")
Expand Down Expand Up @@ -355,6 +395,14 @@ async def _systemone_inner(request: Request):
questions = body["questions"]
_check_request_limits(state, questions)
model = _resolve_model(body.get("model"))
max_budget_cap = _resolve_max_token_budget()
max_len = _validate_budget_param(body, "max_len", max_budget_cap)
head_max_len = _validate_budget_param(body, "head_max_len", max_budget_cap)
predict_kwargs = {}
if max_len is not None:
predict_kwargs["max_len"] = max_len
if head_max_len is not None:
predict_kwargs["head_max_len"] = head_max_len
if gate is None:
gate = asyncio.Lock()
try:
Expand All @@ -363,8 +411,12 @@ async def _systemone_inner(request: Request):
async with gate:
loop = asyncio.get_running_loop()
t0 = time.perf_counter()
result = await loop.run_in_executor(
pool, lambda: router.predict(state, questions, model=model))
if predict_kwargs:
result = await loop.run_in_executor(
pool, lambda: router.predict(state, questions, model=model, **predict_kwargs))
else:
result = await loop.run_in_executor(
pool, lambda: router.predict(state, questions, model=model))
infer_ms = (time.perf_counter() - t0) * 1000.0
from fastapi.responses import JSONResponse
return JSONResponse(
Expand Down
94 changes: 94 additions & 0 deletions tests/test_serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
from fastapi.testclient import TestClient # noqa: E402

from laya.serve import ( # noqa: E402
DEFAULT_MAX_TOKEN_BUDGET,
MAX_BODY_BYTES,
_apply_thread_limit,
_env_bool,
_resolve_max_token_budget,
_resolve_model,
create_app,
)
Expand All @@ -41,6 +43,15 @@ def predict(self, state, questions, model=None):
}


class BudgetRouter(FakeRouter):
"""A router whose predict() takes token-budget keywords and records them."""

def predict(self, state, questions, model=None, **kwargs):
out = super().predict(state, questions, model=model)
self.calls[-1].update(kwargs)
return out


def _client(monkeypatch, api_key=None):
if api_key is None:
monkeypatch.delenv("LAYA_API_KEY", raising=False)
Expand All @@ -50,6 +61,16 @@ def _client(monkeypatch, api_key=None):
return TestClient(create_app(router=fake)), fake


def _budget_client(monkeypatch, api_key=None):
if api_key is None:
monkeypatch.delenv("LAYA_API_KEY", raising=False)
else:
monkeypatch.setenv("LAYA_API_KEY", api_key)
fake = BudgetRouter()
return TestClient(create_app(router=fake)), fake



REQ = {
"model": "jev-1", # a non-Laya model id -> should be ignored, router auto-routes
"state": {"body": "billed twice, refund please"},
Expand Down Expand Up @@ -224,6 +245,7 @@ def test_helpers():


def test_thread_limit(monkeypatch):
pytest.importorskip("torch")
monkeypatch.delenv("LAYA_THREADS", raising=False)
assert _apply_thread_limit() is None # unset -> no-op, no torch import
for bad in ("0", "-4", "abc", ""):
Expand Down Expand Up @@ -400,6 +422,7 @@ def test_a_nested_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch)
`ValueError` to 422, so the caller got a 500 "inference failed" with the reason discarded.
`ValueError` is what carries the message to the client, so the guard has to raise that type.
"""
pytest.importorskip("torch") # laya.agent imports torch at module level
class ValidatingRouter:
"""The real guard, without a checkpoint: what `Agent.system_one` runs before encoding.

Expand Down Expand Up @@ -529,3 +552,74 @@ def test_admission_slot_is_released_after_inference(monkeypatch):
client = TestClient(create_app(router=FakeRouter()))
assert client.post("/v1/systemone", json=REQ).status_code == 200
assert client.post("/v1/systemone", json=REQ).status_code == 200


def test_no_budget_keeps_the_call_unchanged(monkeypatch):
"""An injected router whose predict() takes no kwargs continues to work when body sends no budget."""
client, fake = _client(monkeypatch)
for body in (REQ, dict(REQ, max_len=None, head_max_len=None)):
assert client.post("/v1/systemone", json=body).status_code == 200
assert len(fake.calls) == 2


def test_token_budget_forwarded(monkeypatch):
client, fake = _budget_client(monkeypatch)
r = client.post("/v1/systemone", json={**REQ, "max_len": 4096, "head_max_len": 256})
assert r.status_code == 200
assert fake.calls[0]["max_len"] == 4096
assert fake.calls[0]["head_max_len"] == 256


@pytest.mark.parametrize("bad_budget", ["fast", True, 3.14])
def test_token_budget_validation_type(monkeypatch, bad_budget):
client, fake = _budget_client(monkeypatch)
r = client.post("/v1/systemone", json={**REQ, "max_len": bad_budget})
assert r.status_code == 422
assert "must be an integer" in r.json()["detail"]


@pytest.mark.parametrize("bad_val", [0, -10])
def test_token_budget_validation_positive(monkeypatch, bad_val):
client, fake = _budget_client(monkeypatch)
r = client.post("/v1/systemone", json={**REQ, "max_len": bad_val})
assert r.status_code == 422
assert "must be a positive integer" in r.json()["detail"]


def test_token_budget_exceeds_server_cap(monkeypatch):
client, fake = _budget_client(monkeypatch)
r = client.post("/v1/systemone", json={**REQ, "max_len": 9000})
assert r.status_code == 422
assert "exceeds server limit" in r.json()["detail"]


def test_token_budget_head_max_len_equal_to_max_len(monkeypatch):
"""Core accepts head_max_len == max_len; serve forwards both without artificial restriction."""
client, fake = _budget_client(monkeypatch)
r = client.post("/v1/systemone", json={**REQ, "max_len": 512, "head_max_len": 512})
assert r.status_code == 200
assert fake.calls[0]["max_len"] == 512
assert fake.calls[0]["head_max_len"] == 512


def test_token_budget_env_cap_override(monkeypatch):
monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", "2048")
client, fake = _budget_client(monkeypatch)
r = client.post("/v1/systemone", json={**REQ, "max_len": 4096})
assert r.status_code == 422
assert "exceeds server limit" in r.json()["detail"]

r2 = client.post("/v1/systemone", json={**REQ, "max_len": 2048})
assert r2.status_code == 200
assert fake.calls[0]["max_len"] == 2048


def test_resolve_max_token_budget_fallback(monkeypatch, caplog):
monkeypatch.delenv("LAYA_MAX_TOKEN_BUDGET", raising=False)
assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET
for bad in ("abc", "-10", "0"):
monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", bad)
assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET
assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.text
assert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text