Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions compose.http.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,13 @@ services:
# as 422. The SDK integrations quote 8192 because that is this service's default, so the
# default here stays empty: an operator who sets nothing gets the number the docs print.
LAYA_MAX_TOKEN_BUDGET: "${LAYA_MAX_TOKEN_BUDGET:-}"
# Tokens one `/v1/systemone/batch` forward pass may collate (states x questions x row width);
# a larger batch is split into several passes, not refused. Empty keeps the server default (131072).
LAYA_MAX_BATCH_TOKENS: "${LAYA_MAX_BATCH_TOKENS:-}"
# When set to 1, serve the strict Jev wire contract: no root `routing`, no per-answer
# `action` / `answer_confidence`, no `confidence` on noul answers, and `usage` reduced to
# `input_tokens` + `output_tokens`. Empty keeps the server default (0, full payload).
LAYA_JEV_STRICT: "${LAYA_JEV_STRICT:-}"
LAYA_LOG_LEVEL: "${LAYA_LOG_LEVEL:-info}"
# The supply-chain pair: LAYA_REVISION says which Hub commit a checkpoint may load from (a
# commit, branch, tag, or `reviewed` for the SHAs in laya/revisions.py) and
Expand Down
2 changes: 2 additions & 0 deletions docs/docker.md
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,8 @@ These apply to the `laya-serve` service only.
| `LAYA_API_KEY` | (none) | when set, requires `Authorization: Bearer <key>` |
| `LAYA_ROOT_PATH` | (empty) | public URL prefix for FastAPI when behind a reverse proxy; the proxy should strip it before forwarding |
| `LAYA_MAX_TOKEN_BUDGET` | `8192` | cap on per-request `max_len` and `head_max_len` overrides |
| `LAYA_MAX_BATCH_TOKENS` | `131072` | tokens one `/v1/systemone/batch` forward pass may collate (`states` × questions × row width); a larger batch is split into several passes, not refused |
| `LAYA_JEV_STRICT` | `0` | `1` serves the strict Jev wire contract (no root `routing`, no per-answer `action` / `answer_confidence`, reduced `usage`) for strict Jev clients |
| `LAYA_SHA256_DIGESTS` | (none) | JSON digests checked before a checkpoint is parsed: `{artifact: digest}` for every checkpoint, or `{model: {artifact: digest}}` per checkpoint. See [Security](security.md) |

For example, set `LAYA_ROOT_PATH=/laya` when publishing the API under `/laya`. The proxy must
Expand Down
6 changes: 4 additions & 2 deletions laya-ts/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,8 @@ export class Agent extends HookRegistry {
? [...this.hooks, ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd)]
: composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd);
const raiseErrors = opts.hooksRaise ?? this.hooksRaise;
const hooksTimeout = opts.hooksTimeout ?? this.hooksTimeout;
const timeoutMs = typeof hooksTimeout === "number" ? hooksTimeout * 1000 : null;
const ctx = new PredictContext({
states,
questions: questions as Record<string, unknown>,
Expand All @@ -524,7 +526,7 @@ export class Agent extends HookRegistry {
headMaxLen: opts.headMaxLen ?? null,
});
try {
await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null });
await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs });
if (ctx.results === null) {
if (!Array.isArray(ctx.states)) {
throw new TypeError(
Expand Down Expand Up @@ -575,7 +577,7 @@ export class Agent extends HookRegistry {
}
}
try {
await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null });
await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs });
} catch (hookErr) {
// End hooks run on the failure path too; do not let one mask the real error.
if (ctx.error === null) throw hookErr;
Expand Down
30 changes: 18 additions & 12 deletions laya-ts/src/hooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -335,22 +335,28 @@ function reportHookFailure(hook: Hook, event: HookEvent, err: unknown): void {
*/

/** Run a promise with a timeout, returning void if it completes or throwing on timeout. */
async function runWithTimeout<T>(
promise: Promise<T>,
async function runWithTimeout(
promise: unknown,
timeoutMs: number | null,
operationName: string,
): Promise<T> {
if (timeoutMs === null || timeoutMs === undefined) return promise;
): Promise<void> {
if (timeoutMs === null || timeoutMs === undefined) {
await promise;
return;
}

const abortController = new AbortController();
const timeoutId = setTimeout(() => abortController.abort(), timeoutMs);

try {
return await Promise.race([promise, new Promise<never>((_, reject) => {
abortController.signal.addEventListener("abort", () => {
reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms")));
});
})]);
await Promise.race([
Promise.resolve(promise),
new Promise<never>((_, reject) => {
abortController.signal.addEventListener("abort", () => {
reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms`));
});
}),
]);
} finally {
clearTimeout(timeoutId);
}
Expand All @@ -360,10 +366,10 @@ export function dispatch(
hooks: Hook[],
event: HookEvent,
ctx: PredictContext,
opts: { raiseErrors?: boolean; timeoutMs?: number | null } = {},
opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {},
): void {
const timeoutMs = opts.timeoutMs ?? null;
const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`;
const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`;
const raiseErrors = opts.raiseErrors ?? true;
for (const hook of hooks) {
const method = hook?.[event];
Expand All @@ -389,7 +395,7 @@ export async function dispatchAsync(
opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {},
): Promise<void> {
const timeoutMs = opts.timeoutMs ?? null;
const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`;
const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`;
const raiseErrors = opts.raiseErrors ?? true;
for (const hook of hooks) {
const method = hook?.[event];
Expand Down
1 change: 1 addition & 0 deletions laya/serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
a ``["repo", "subfolder"]`` pair. Served under
explicit ``model=`` like a built-in.
``LAYA_MAX_TOKEN_BUDGET`` cap on per-request max_len / head_max_len 8192
``LAYA_MAX_BATCH_TOKENS`` cap on batch forward tokens (states x q x len) 131072
``LAYA_JEV_STRICT`` if set, serve the strict Jev wire contract: no 0
root `routing`, no per-answer `action` /
`answer_confidence`, no `confidence` on noul
Expand Down
15 changes: 9 additions & 6 deletions tests/test_compose_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,8 @@ def pins(text: str) -> List[tuple]:
_env = env_blocks(read("compose.http.yaml")).get(_service, {})
check_true("compose/compose.http.yaml defines the %s service" % _service,
_service in env_blocks(read("compose.http.yaml")), sorted(_env))
for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION"):
for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION",
"LAYA_MAX_BATCH_TOKENS", "LAYA_JEV_STRICT"):
check_true("compose/compose.http.yaml/%s forwards the %s this change adds"
% (_service, _name), _name in _env,
"it is documented and read by the runtime, and the service that needs it "
Expand Down Expand Up @@ -579,9 +580,9 @@ def pins(text: str) -> List[tuple]:
_const_pinned += 1
check("docs/docker.md prints the same %s default as laya/serve.py" % _name,
_cell.strip("`"), _defaults[_const])
check_true("core/both knobs this change forwards have a module default to pin",
_const_pinned == 2, _const_pinned)
for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET"):
check_true("core/knobs this change forwards have a module default to pin",
_const_pinned == 3, _const_pinned)
for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_MAX_BATCH_TOKENS"):
check_true("docs/docker.md keeps the %s row in the server table" % _name,
re.search(r"^\| `%s` \|" % _name, _body, re.M) is not None,
"the two sections above derive their demand from this table, so a row "
Expand Down Expand Up @@ -610,8 +611,10 @@ def pins(text: str) -> List[tuple]:
# container is the failure the textual checks above are reasoning about. Both arms: set, and
# left alone. The unset arm is the one an operator actually boots, and it must show the empty
# default arriving as empty rather than as the literal `${LAYA_MAX_CONCURRENT:-}`.
_KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION")
_arms = [(dict(zip(_KN, ("4", "16384", "reviewed"))), dict(zip(_KN, ("4", "16384", "reviewed")))),
_KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION",
"LAYA_MAX_BATCH_TOKENS", "LAYA_JEV_STRICT")
_arms = [(dict(zip(_KN, ("4", "16384", "reviewed", "65536", "1"))),
dict(zip(_KN, ("4", "16384", "reviewed", "65536", "1")))),
({}, dict.fromkeys(_KN, ""))]
for _set, _expect in _arms:
proc = subprocess.run(["docker", "compose", "-f", "compose.yaml", "-f", "compose.http.yaml",
Expand Down
Loading