From 5067f79e611ecc67db3d38e681703de7471add8d Mon Sep 17 00:00:00 2001 From: naveenprasaath23 Date: Fri, 9 Oct 2026 00:08:36 +0530 Subject: [PATCH 1/2] fix(compose): forward extra models and idle unload knobs to laya-serve --- compose.http.yaml | 6 ++++++ docs/docker.md | 2 ++ tests/test_compose_env.py | 9 ++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/compose.http.yaml b/compose.http.yaml index 7d85a4f11..1e4d56b45 100644 --- a/compose.http.yaml +++ b/compose.http.yaml @@ -51,6 +51,12 @@ services: # default (english); set `multilingual` for mostly-non-English traffic. An unresolvable # name stops the container at startup rather than serving a configuration nobody asked for. LAYA_DEFAULT_MODEL: "${LAYA_DEFAULT_MODEL:-}" + # Extra checkpoints beside the bundled ones, as a JSON object {name: source}. Empty registers + # nothing; a malformed value or bad name stops the container at startup. + LAYA_EXTRA_MODELS: "${LAYA_EXTRA_MODELS:-}" + # Unload resident checkpoints after this many idle seconds. Empty keeps the server default (0, + # disabled); the next request loads its checkpoint again. + LAYA_IDLE_UNLOAD_SECONDS: "${LAYA_IDLE_UNLOAD_SECONDS:-}" # Empty keeps Router's own default (2). Set 3 when LAYA_AUTO_TASK is on: that adds a # third checkpoint routing can choose on demand, and a cap below it reloads one per switch. LAYA_MAX_LOADED: "${LAYA_MAX_LOADED:-}" diff --git a/docs/docker.md b/docs/docker.md index dcabf6495..a872f43e3 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -339,6 +339,8 @@ These apply to the `laya-serve` service only. | `LAYA_THREADS` | `OMP_NUM_THREADS` | caps torch intra-op threads; keep at or below physical cores | | `LAYA_AUTO_TASK` | `0` | `1` lets the router reach `typed-decisions` automatically | | `LAYA_DEFAULT_MODEL` | `multilingual` | Checkpoint a state with no language evidence falls back to (no letters, or Latin text too short to identify). Set `english` for mostly English traffic; an unresolvable name stops the container at startup instead of serving a configuration nobody asked for | +| `LAYA_EXTRA_MODELS` | (none) | JSON object `{name: source}` registering extra checkpoints beside the bundled ones: a Hub repo id or local checkpoint directory as a string, or a `["repo", "subfolder"]` pair | +| `LAYA_IDLE_UNLOAD_SECONDS` | `0` | unload resident checkpoints after this many idle seconds; the next request loads its checkpoint again. Zero disables unloading | | `LAYA_MAX_LOADED` | `2` | Checkpoints kept resident; `LAYA_AUTO_TASK` makes a third reachable on demand, and a cap below what routing chooses rebuilds one per switch | | `LAYA_MAX_CONCURRENT` | `16` | requests admitted at once; later ones get `503` (a value that does not parse, or is not positive, falls back to `16`) | | `LAYA_LOG_LEVEL` | `info` | uvicorn log level | diff --git a/tests/test_compose_env.py b/tests/test_compose_env.py index a3152016a..be435448e 100644 --- a/tests/test_compose_env.py +++ b/tests/test_compose_env.py @@ -537,7 +537,8 @@ def pins(text: str) -> List[tuple]: _env = env_blocks(read("compose.http.yaml")).get(_service, {}) check_true("compose/compose.http.yaml defines the %s service" % _service, _service in env_blocks(read("compose.http.yaml")), sorted(_env)) - for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION"): + for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION", + "LAYA_EXTRA_MODELS", "LAYA_IDLE_UNLOAD_SECONDS"): check_true("compose/compose.http.yaml/%s forwards the %s this change adds" % (_service, _name), _name in _env, "it is documented and read by the runtime, and the service that needs it " @@ -610,8 +611,10 @@ def pins(text: str) -> List[tuple]: # container is the failure the textual checks above are reasoning about. Both arms: set, and # left alone. The unset arm is the one an operator actually boots, and it must show the empty # default arriving as empty rather than as the literal `${LAYA_MAX_CONCURRENT:-}`. - _KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION") - _arms = [(dict(zip(_KN, ("4", "16384", "reviewed"))), dict(zip(_KN, ("4", "16384", "reviewed")))), + _KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION", + "LAYA_EXTRA_MODELS", "LAYA_IDLE_UNLOAD_SECONDS") + _arms = [(dict(zip(_KN, ("4", "16384", "reviewed", '{"custom":"./model"}', "30"))), + dict(zip(_KN, ("4", "16384", "reviewed", '{"custom":"./model"}', "30")))), ({}, dict.fromkeys(_KN, ""))] for _set, _expect in _arms: proc = subprocess.run(["docker", "compose", "-f", "compose.yaml", "-f", "compose.http.yaml", From acc23b0cd752c51ea0070a4571845c1535dde9b9 Mon Sep 17 00:00:00 2001 From: naveenprasaath23 Date: Fri, 9 Oct 2026 00:56:47 +0530 Subject: [PATCH 2/2] fix(ts): resolve TypeScript compilation errors in hooks timeout runner --- laya-ts/src/agent.ts | 6 ++++-- laya-ts/src/hooks.ts | 30 ++++++++++++++++++------------ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts index a895ff128..dfa8077ea 100644 --- a/laya-ts/src/agent.ts +++ b/laya-ts/src/agent.ts @@ -516,6 +516,8 @@ export class Agent extends HookRegistry { ? [...this.hooks, ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd)] : composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd); const raiseErrors = opts.hooksRaise ?? this.hooksRaise; + const hooksTimeout = opts.hooksTimeout ?? this.hooksTimeout; + const timeoutMs = typeof hooksTimeout === "number" ? hooksTimeout * 1000 : null; const ctx = new PredictContext({ states, questions: questions as Record, @@ -524,7 +526,7 @@ export class Agent extends HookRegistry { headMaxLen: opts.headMaxLen ?? null, }); try { - await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null }); + await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs }); if (ctx.results === null) { if (!Array.isArray(ctx.states)) { throw new TypeError( @@ -575,7 +577,7 @@ export class Agent extends HookRegistry { } } try { - await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null }); + await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs }); } catch (hookErr) { // End hooks run on the failure path too; do not let one mask the real error. if (ctx.error === null) throw hookErr; diff --git a/laya-ts/src/hooks.ts b/laya-ts/src/hooks.ts index de475e2a1..412c9f78c 100644 --- a/laya-ts/src/hooks.ts +++ b/laya-ts/src/hooks.ts @@ -335,22 +335,28 @@ function reportHookFailure(hook: Hook, event: HookEvent, err: unknown): void { */ /** Run a promise with a timeout, returning void if it completes or throwing on timeout. */ -async function runWithTimeout( - promise: Promise, +async function runWithTimeout( + promise: unknown, timeoutMs: number | null, operationName: string, -): Promise { - if (timeoutMs === null || timeoutMs === undefined) return promise; +): Promise { + if (timeoutMs === null || timeoutMs === undefined) { + await promise; + return; + } const abortController = new AbortController(); const timeoutId = setTimeout(() => abortController.abort(), timeoutMs); try { - return await Promise.race([promise, new Promise((_, reject) => { - abortController.signal.addEventListener("abort", () => { - reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms"))); - }); - })]); + await Promise.race([ + Promise.resolve(promise), + new Promise((_, reject) => { + abortController.signal.addEventListener("abort", () => { + reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms`)); + }); + }), + ]); } finally { clearTimeout(timeoutId); } @@ -360,10 +366,10 @@ export function dispatch( hooks: Hook[], event: HookEvent, ctx: PredictContext, - opts: { raiseErrors?: boolean; timeoutMs?: number | null } = {}, + opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {}, ): void { const timeoutMs = opts.timeoutMs ?? null; - const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`; + const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`; const raiseErrors = opts.raiseErrors ?? true; for (const hook of hooks) { const method = hook?.[event]; @@ -389,7 +395,7 @@ export async function dispatchAsync( opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {}, ): Promise { const timeoutMs = opts.timeoutMs ?? null; - const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`; + const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`; const raiseErrors = opts.raiseErrors ?? true; for (const hook of hooks) { const method = hook?.[event];