diff --git a/compose.http.yaml b/compose.http.yaml index 7d85a4f11..580cb5bff 100644 --- a/compose.http.yaml +++ b/compose.http.yaml @@ -62,6 +62,13 @@ services: # as 422. The SDK integrations quote 8192 because that is this service's default, so the # default here stays empty: an operator who sets nothing gets the number the docs print. LAYA_MAX_TOKEN_BUDGET: "${LAYA_MAX_TOKEN_BUDGET:-}" + # Tokens one `/v1/systemone/batch` forward pass may collate (states x questions x row width); + # a larger batch is split into several passes, not refused. Empty keeps the server default (131072). + LAYA_MAX_BATCH_TOKENS: "${LAYA_MAX_BATCH_TOKENS:-}" + # When set to 1, serve the strict Jev wire contract: no root `routing`, no per-answer + # `action` / `answer_confidence`, no `confidence` on noul answers, and `usage` reduced to + # `input_tokens` + `output_tokens`. Empty keeps the server default (0, full payload). + LAYA_JEV_STRICT: "${LAYA_JEV_STRICT:-}" LAYA_LOG_LEVEL: "${LAYA_LOG_LEVEL:-info}" # The supply-chain pair: LAYA_REVISION says which Hub commit a checkpoint may load from (a # commit, branch, tag, or `reviewed` for the SHAs in laya/revisions.py) and diff --git a/docs/docker.md b/docs/docker.md index dcabf6495..d4d4c1154 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -345,6 +345,8 @@ These apply to the `laya-serve` service only. | `LAYA_API_KEY` | (none) | when set, requires `Authorization: Bearer ` | | `LAYA_ROOT_PATH` | (empty) | public URL prefix for FastAPI when behind a reverse proxy; the proxy should strip it before forwarding | | `LAYA_MAX_TOKEN_BUDGET` | `8192` | cap on per-request `max_len` and `head_max_len` overrides | +| `LAYA_MAX_BATCH_TOKENS` | `131072` | tokens one `/v1/systemone/batch` forward pass may collate (`states` × questions × row width); a larger batch is split into several passes, not refused | +| `LAYA_JEV_STRICT` | `0` | `1` serves the strict Jev wire contract (no root `routing`, no per-answer `action` / `answer_confidence`, reduced `usage`) for strict Jev clients | | `LAYA_SHA256_DIGESTS` | (none) | JSON digests checked before a checkpoint is parsed: `{artifact: digest}` for every checkpoint, or `{model: {artifact: digest}}` per checkpoint. See [Security](security.md) | For example, set `LAYA_ROOT_PATH=/laya` when publishing the API under `/laya`. The proxy must diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts index a895ff128..dfa8077ea 100644 --- a/laya-ts/src/agent.ts +++ b/laya-ts/src/agent.ts @@ -516,6 +516,8 @@ export class Agent extends HookRegistry { ? [...this.hooks, ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd)] : composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd); const raiseErrors = opts.hooksRaise ?? this.hooksRaise; + const hooksTimeout = opts.hooksTimeout ?? this.hooksTimeout; + const timeoutMs = typeof hooksTimeout === "number" ? hooksTimeout * 1000 : null; const ctx = new PredictContext({ states, questions: questions as Record, @@ -524,7 +526,7 @@ export class Agent extends HookRegistry { headMaxLen: opts.headMaxLen ?? null, }); try { - await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null }); + await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs }); if (ctx.results === null) { if (!Array.isArray(ctx.states)) { throw new TypeError( @@ -575,7 +577,7 @@ export class Agent extends HookRegistry { } } try { - await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null }); + await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs }); } catch (hookErr) { // End hooks run on the failure path too; do not let one mask the real error. if (ctx.error === null) throw hookErr; diff --git a/laya-ts/src/hooks.ts b/laya-ts/src/hooks.ts index de475e2a1..412c9f78c 100644 --- a/laya-ts/src/hooks.ts +++ b/laya-ts/src/hooks.ts @@ -335,22 +335,28 @@ function reportHookFailure(hook: Hook, event: HookEvent, err: unknown): void { */ /** Run a promise with a timeout, returning void if it completes or throwing on timeout. */ -async function runWithTimeout( - promise: Promise, +async function runWithTimeout( + promise: unknown, timeoutMs: number | null, operationName: string, -): Promise { - if (timeoutMs === null || timeoutMs === undefined) return promise; +): Promise { + if (timeoutMs === null || timeoutMs === undefined) { + await promise; + return; + } const abortController = new AbortController(); const timeoutId = setTimeout(() => abortController.abort(), timeoutMs); try { - return await Promise.race([promise, new Promise((_, reject) => { - abortController.signal.addEventListener("abort", () => { - reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms"))); - }); - })]); + await Promise.race([ + Promise.resolve(promise), + new Promise((_, reject) => { + abortController.signal.addEventListener("abort", () => { + reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms`)); + }); + }), + ]); } finally { clearTimeout(timeoutId); } @@ -360,10 +366,10 @@ export function dispatch( hooks: Hook[], event: HookEvent, ctx: PredictContext, - opts: { raiseErrors?: boolean; timeoutMs?: number | null } = {}, + opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {}, ): void { const timeoutMs = opts.timeoutMs ?? null; - const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`; + const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`; const raiseErrors = opts.raiseErrors ?? true; for (const hook of hooks) { const method = hook?.[event]; @@ -389,7 +395,7 @@ export async function dispatchAsync( opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {}, ): Promise { const timeoutMs = opts.timeoutMs ?? null; - const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`; + const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`; const raiseErrors = opts.raiseErrors ?? true; for (const hook of hooks) { const method = hook?.[event]; diff --git a/laya/serve.py b/laya/serve.py index 8706b9fd3..c68c4ca91 100644 --- a/laya/serve.py +++ b/laya/serve.py @@ -41,6 +41,7 @@ a ``["repo", "subfolder"]`` pair. Served under explicit ``model=`` like a built-in. ``LAYA_MAX_TOKEN_BUDGET`` cap on per-request max_len / head_max_len 8192 +``LAYA_MAX_BATCH_TOKENS`` cap on batch forward tokens (states x q x len) 131072 ``LAYA_JEV_STRICT`` if set, serve the strict Jev wire contract: no 0 root `routing`, no per-answer `action` / `answer_confidence`, no `confidence` on noul diff --git a/tests/test_compose_env.py b/tests/test_compose_env.py index a3152016a..4e8dcee77 100644 --- a/tests/test_compose_env.py +++ b/tests/test_compose_env.py @@ -537,7 +537,8 @@ def pins(text: str) -> List[tuple]: _env = env_blocks(read("compose.http.yaml")).get(_service, {}) check_true("compose/compose.http.yaml defines the %s service" % _service, _service in env_blocks(read("compose.http.yaml")), sorted(_env)) - for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION"): + for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION", + "LAYA_MAX_BATCH_TOKENS", "LAYA_JEV_STRICT"): check_true("compose/compose.http.yaml/%s forwards the %s this change adds" % (_service, _name), _name in _env, "it is documented and read by the runtime, and the service that needs it " @@ -579,9 +580,9 @@ def pins(text: str) -> List[tuple]: _const_pinned += 1 check("docs/docker.md prints the same %s default as laya/serve.py" % _name, _cell.strip("`"), _defaults[_const]) - check_true("core/both knobs this change forwards have a module default to pin", - _const_pinned == 2, _const_pinned) - for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET"): + check_true("core/knobs this change forwards have a module default to pin", + _const_pinned == 3, _const_pinned) + for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_MAX_BATCH_TOKENS"): check_true("docs/docker.md keeps the %s row in the server table" % _name, re.search(r"^\| `%s` \|" % _name, _body, re.M) is not None, "the two sections above derive their demand from this table, so a row " @@ -610,8 +611,10 @@ def pins(text: str) -> List[tuple]: # container is the failure the textual checks above are reasoning about. Both arms: set, and # left alone. The unset arm is the one an operator actually boots, and it must show the empty # default arriving as empty rather than as the literal `${LAYA_MAX_CONCURRENT:-}`. - _KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION") - _arms = [(dict(zip(_KN, ("4", "16384", "reviewed"))), dict(zip(_KN, ("4", "16384", "reviewed")))), + _KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION", + "LAYA_MAX_BATCH_TOKENS", "LAYA_JEV_STRICT") + _arms = [(dict(zip(_KN, ("4", "16384", "reviewed", "65536", "1"))), + dict(zip(_KN, ("4", "16384", "reviewed", "65536", "1")))), ({}, dict.fromkeys(_KN, ""))] for _set, _expect in _arms: proc = subprocess.run(["docker", "compose", "-f", "compose.yaml", "-f", "compose.http.yaml",