Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/docker.md
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,13 @@ The service has a healthcheck on `/health`. The server preloads before it starts
listening, so with `LAYA_PRELOAD=1` a healthy container has its checkpoints loaded.
`docker compose ... up -d --wait laya-serve` returns once it is healthy.

`/health` reports `device` as the device a resident checkpoint actually computes
on, which is not always what `LAYA_DEVICE` asked for: a checkpoint that wants a
GPU it cannot get falls back to CPU silently and still answers correctly.
`checkpoint_devices` names each loaded checkpoint, and `device_is_preference` is
`true` only while nothing is resident, so a deployment that quietly lost its GPU
says so instead of echoing its own configuration back.

### Server configuration

These apply to the `laya-serve` service only.
Expand Down
28 changes: 26 additions & 2 deletions laya/serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
gets 503 (see below)
====================== ============================================ =========

``LAYA_DEVICE`` is a preference, not a guarantee: an ``Agent`` that asks for a
GPU it cannot get falls back to CPU silently and keeps answering correctly.
``/health`` reports where a resident checkpoint actually computes, using the
same helpers the MCP status tool uses (``laya.mcp.device``).

Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are all
deferred into the functions that need them, so ``import laya.serve`` stays cheap
and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.
Expand Down Expand Up @@ -229,10 +234,14 @@ def _apply_thread_limit():

def build_router():
"""Build a Router from the environment, preloading unless told otherwise."""
from .mcp.device import env_device
from .router import Router

_apply_thread_limit()
device = os.environ.get("LAYA_DEVICE") or None
# `env_device` rather than a raw read: it strips, the way `LAYA_MODELS` two lines
# below is stripped. A value copied out of a Dockerfile or a `.env` file carries a
# trailing newline, and torch refuses that as an invalid device string.
device = env_device()
models_env = os.environ.get("LAYA_MODELS", "").strip()
preload_names = [m.strip() for m in models_env.split(",") if m.strip()] or None
router = Router(device=device, auto_task_detection=_env_bool("LAYA_AUTO_TASK", False))
Expand All @@ -249,6 +258,8 @@ def create_app(router: Optional[Any] = None):

from fastapi import FastAPI, Header, HTTPException, Request

from .mcp.device import agent_device, resolve_device, router_agent

if router is None:
router = build_router()
api_key = os.environ.get("LAYA_API_KEY") or None
Expand Down Expand Up @@ -306,11 +317,24 @@ def _check_auth(authorization: Optional[str]) -> None:

@app.get("/health")
def health() -> Dict[str, Any]:
# `device` is where a resident checkpoint really computes, not what was asked for:
# `Agent.device` reflects the silent GPU -> CPU fallback, so a container that asked
# for a GPU it did not get says so. With nothing resident it is the configured
# preference, and `device_is_preference` tells the reader which of the two it is
# looking at. Same convention, and the same three keys, as `laya_status` over MCP.
checkpoint_devices: Dict[str, str] = {}
for name in (router.loaded or []):
device = agent_device(router_agent(router, name))
if device:
checkpoint_devices[name] = device
actual = next(iter(checkpoint_devices.values()), None)
return {
"status": "ok",
"loaded": router.loaded,
"revisions": getattr(router, "loaded_revisions", {}),
"device": os.environ.get("LAYA_DEVICE") or "auto",
"device": actual or resolve_device(),
"device_is_preference": actual is None,
"checkpoint_devices": checkpoint_devices,
}

@app.post("/v1/systemone")
Expand Down
119 changes: 119 additions & 0 deletions tests/test_serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,18 @@
"""
import json
import logging
import os
import subprocess
import sys
from types import SimpleNamespace

import pytest

fastapi = pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

from laya.serve import ( # noqa: E402
MAX_BODY_BYTES,
_apply_thread_limit,
Expand Down Expand Up @@ -201,6 +207,119 @@ def test_body_read_preserves_parse_error_codes(monkeypatch):
assert r.status_code == 400, (payload, r.status_code)


class DeviceRouter:
"""A Router with resident checkpoints whose real devices are known.

`Agent.device` is a `torch.device`; `laya.mcp.device.agent_device` also accepts the
plain string, which keeps this stub free of torch and of checkpoint weights.
"""

def __init__(self, **devices):
self._agents = {name: SimpleNamespace(device=device)
for name, device in devices.items()}
self.loaded = list(devices)

def predict(self, state, questions, model=None):
return {"model": "laya-rl-agent",
"answers": {"dept": {"type": "choice", "choice": "billing",
"probabilities": {"billing": 1.0}, "confidence": 1.0}},
"usage": {"input_tokens": 1, "output_tokens": 0},
"routing": {"model": (self.loaded or ["english"])[0]}}


def test_health_reports_where_inference_actually_runs(monkeypatch):
"""`LAYA_DEVICE` is a request, not a fact: the Agent falls back to CPU silently."""
monkeypatch.setenv("LAYA_DEVICE", "cuda")
fake = DeviceRouter(english="cpu") # asked for cuda, ended up on cpu
body = TestClient(create_app(router=fake)).get("/health").json()
assert body["device"] == "cpu", body
assert body["device_is_preference"] is False, body
assert body["checkpoint_devices"] == {"english": "cpu"}, body


def test_health_names_each_checkpoint_device(monkeypatch):
monkeypatch.setenv("LAYA_DEVICE", "cuda")
fake = DeviceRouter(english="cpu", multilingual="cuda")
body = TestClient(create_app(router=fake)).get("/health").json()
assert body["checkpoint_devices"] == {"english": "cpu", "multilingual": "cuda"}, body
# The top-level answer is the first resident one, exactly as `laya_status` reports it.
assert body["device"] == body["checkpoint_devices"][body["loaded"][0]], body


def test_health_without_a_resident_checkpoint_flags_the_preference(monkeypatch):
monkeypatch.setenv("LAYA_DEVICE", "cuda")
body = TestClient(create_app(router=FakeRouter())).get("/health").json()
assert body["device_is_preference"] is True, body
assert body["checkpoint_devices"] == {}, body
assert body["device"] == "cuda", body # what was asked for, labelled as such


def test_health_agrees_with_the_mcp_status_tool(monkeypatch):
"""One fact about the device, reported the same way by both surfaces."""
pytest.importorskip("mcp")
from laya.mcp.tools import laya_status

monkeypatch.setenv("LAYA_DEVICE", "cuda")
fake = DeviceRouter(english="cpu")
body = TestClient(create_app(router=fake)).get("/health").json()
status = laya_status(router=fake, loaded=list(fake.loaded))
assert body["device"] == status["device"], (body["device"], status["device"])
assert body["checkpoint_devices"] == status["checkpoint_devices"], body
assert body["device_is_preference"] == status["device_is_preference"], body


def test_health_needs_neither_the_mcp_extra_nor_torch():
"""`laya.mcp.device` is documented as importable without `mcp`; prove the server agrees."""
probe = r'''
import sys
class Blocker:
def find_spec(self, name, path=None, target=None):
if name == "mcp" or name.startswith("mcp."):
raise ImportError("mcp blocked")
sys.meta_path.insert(0, Blocker())
sys.path.insert(0, %r)
from laya.mcp.device import agent_device, env_device, resolve_device, router_agent
assert agent_device(type("A", (), {"device": "cpu"})()) == "cpu"
assert env_device.__module__ == "laya.mcp.device"
import laya.serve
assert "mcp" not in sys.modules, "laya.mcp.device reached the mcp distribution"
from fastapi.testclient import TestClient
client = TestClient(laya.serve.create_app(router=type("R", (), {"loaded": ["english"],
"_agents": {"english": type("A", (), {"device": "cpu"})()}})()))
body = client.get("/health").json()
assert body["device"] == "cpu" and body["checkpoint_devices"] == {"english": "cpu"}, body
print("ok")
''' % ROOT
out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True)
assert out.returncode == 0, out.stderr[-2000:]
assert "ok" in out.stdout, out.stdout


def test_build_router_strips_the_device_before_torch_sees_it(monkeypatch):
"""A value pasted from a Dockerfile or a `.env` file carries a trailing newline."""
import laya.router
import laya.serve

seen = {}

class RecordingRouter:
def __init__(self, device=None, **kwargs):
seen["device"] = device

def preload(self, names=None):
seen["preloaded"] = names

monkeypatch.setattr(laya.router, "Router", RecordingRouter)
monkeypatch.setenv("LAYA_PRELOAD", "0")
for raw, want in ((" cpu\n", "cpu"), ("cuda ", "cuda"), (" ", None), ("cpu", "cpu")):
monkeypatch.setenv("LAYA_DEVICE", raw)
laya.serve.build_router()
assert seen["device"] == want, "%r -> %r" % (raw, seen["device"])
monkeypatch.delenv("LAYA_DEVICE")
laya.serve.build_router()
assert seen["device"] is None, repr(seen["device"]) # unset means auto


def test_health_supports_router_without_loaded_revisions(monkeypatch):
# FakeRouter deliberately has no loaded_revisions attribute. Injected test or
# embedding routers predating revision reporting must remain health-compatible.
Expand Down
Loading