Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions compose.http.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ services:
# default (english); set `multilingual` for mostly-non-English traffic. An unresolvable
# name stops the container at startup rather than serving a configuration nobody asked for.
LAYA_DEFAULT_MODEL: "${LAYA_DEFAULT_MODEL:-}"
# Extra checkpoints beside the bundled ones, as a JSON object {name: source}. Empty registers
# nothing; a malformed value or bad name stops the container at startup.
LAYA_EXTRA_MODELS: "${LAYA_EXTRA_MODELS:-}"
# Unload resident checkpoints after this many idle seconds. Empty keeps the server default (0,
# disabled); the next request loads its checkpoint again.
LAYA_IDLE_UNLOAD_SECONDS: "${LAYA_IDLE_UNLOAD_SECONDS:-}"
# Empty keeps Router's own default (2). Set 3 when LAYA_AUTO_TASK is on: that adds a
# third checkpoint routing can choose on demand, and a cap below it reloads one per switch.
LAYA_MAX_LOADED: "${LAYA_MAX_LOADED:-}"
Expand Down
2 changes: 2 additions & 0 deletions docs/docker.md
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,8 @@ These apply to the `laya-serve` service only.
| `LAYA_THREADS` | `OMP_NUM_THREADS` | caps torch intra-op threads; keep at or below physical cores |
| `LAYA_AUTO_TASK` | `0` | `1` lets the router reach `typed-decisions` automatically |
| `LAYA_DEFAULT_MODEL` | `multilingual` | Checkpoint a state with no language evidence falls back to (no letters, or Latin text too short to identify). Set `english` for mostly English traffic; an unresolvable name stops the container at startup instead of serving a configuration nobody asked for |
| `LAYA_EXTRA_MODELS` | (none) | JSON object `{name: source}` registering extra checkpoints beside the bundled ones: a Hub repo id or local checkpoint directory as a string, or a `["repo", "subfolder"]` pair |
| `LAYA_IDLE_UNLOAD_SECONDS` | `0` | unload resident checkpoints after this many idle seconds; the next request loads its checkpoint again. Zero disables unloading |
| `LAYA_MAX_LOADED` | `2` | Checkpoints kept resident; `LAYA_AUTO_TASK` makes a third reachable on demand, and a cap below what routing chooses rebuilds one per switch |
| `LAYA_MAX_CONCURRENT` | `16` | requests admitted at once; later ones get `503` (a value that does not parse, or is not positive, falls back to `16`) |
| `LAYA_LOG_LEVEL` | `info` | uvicorn log level |
Expand Down
6 changes: 4 additions & 2 deletions laya-ts/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,8 @@ export class Agent extends HookRegistry {
? [...this.hooks, ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd)]
: composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd);
const raiseErrors = opts.hooksRaise ?? this.hooksRaise;
const hooksTimeout = opts.hooksTimeout ?? this.hooksTimeout;
const timeoutMs = typeof hooksTimeout === "number" ? hooksTimeout * 1000 : null;
const ctx = new PredictContext({
states,
questions: questions as Record<string, unknown>,
Expand All @@ -524,7 +526,7 @@ export class Agent extends HookRegistry {
headMaxLen: opts.headMaxLen ?? null,
});
try {
await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null });
await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors, timeoutMs });
if (ctx.results === null) {
if (!Array.isArray(ctx.states)) {
throw new TypeError(
Expand Down Expand Up @@ -575,7 +577,7 @@ export class Agent extends HookRegistry {
}
}
try {
await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs: this.hooksTimeout !== null ? this.hooksTimeout * 1000 : null });
await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors, timeoutMs });
} catch (hookErr) {
// End hooks run on the failure path too; do not let one mask the real error.
if (ctx.error === null) throw hookErr;
Expand Down
30 changes: 18 additions & 12 deletions laya-ts/src/hooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -335,22 +335,28 @@ function reportHookFailure(hook: Hook, event: HookEvent, err: unknown): void {
*/

/** Run a promise with a timeout, returning void if it completes or throwing on timeout. */
async function runWithTimeout<T>(
promise: Promise<T>,
async function runWithTimeout(
promise: unknown,
timeoutMs: number | null,
operationName: string,
): Promise<T> {
if (timeoutMs === null || timeoutMs === undefined) return promise;
): Promise<void> {
if (timeoutMs === null || timeoutMs === undefined) {
await promise;
return;
}

const abortController = new AbortController();
const timeoutId = setTimeout(() => abortController.abort(), timeoutMs);

try {
return await Promise.race([promise, new Promise<never>((_, reject) => {
abortController.signal.addEventListener("abort", () => {
reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms")));
});
})]);
await Promise.race([
Promise.resolve(promise),
new Promise<never>((_, reject) => {
abortController.signal.addEventListener("abort", () => {
reject(new Error(`hook ${operationName} timed out after ${timeoutMs}ms`));
});
}),
]);
} finally {
clearTimeout(timeoutId);
}
Expand All @@ -360,10 +366,10 @@ export function dispatch(
hooks: Hook[],
event: HookEvent,
ctx: PredictContext,
opts: { raiseErrors?: boolean; timeoutMs?: number | null } = {},
opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {},
): void {
const timeoutMs = opts.timeoutMs ?? null;
const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`;
const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`;
const raiseErrors = opts.raiseErrors ?? true;
for (const hook of hooks) {
const method = hook?.[event];
Expand All @@ -389,7 +395,7 @@ export async function dispatchAsync(
opts: { raiseErrors?: boolean; timeoutMs?: number | null; hook?: Hook } = {},
): Promise<void> {
const timeoutMs = opts.timeoutMs ?? null;
const operationName = `${(opts.hook?.constructor?.name ?? "hook")}.`;
const operationName = `${((opts.hook as object | undefined)?.constructor?.name ?? "hook")}.`;
const raiseErrors = opts.raiseErrors ?? true;
for (const hook of hooks) {
const method = hook?.[event];
Expand Down
9 changes: 6 additions & 3 deletions tests/test_compose_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,8 @@ def pins(text: str) -> List[tuple]:
_env = env_blocks(read("compose.http.yaml")).get(_service, {})
check_true("compose/compose.http.yaml defines the %s service" % _service,
_service in env_blocks(read("compose.http.yaml")), sorted(_env))
for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION"):
for _name in ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION",
"LAYA_EXTRA_MODELS", "LAYA_IDLE_UNLOAD_SECONDS"):
check_true("compose/compose.http.yaml/%s forwards the %s this change adds"
% (_service, _name), _name in _env,
"it is documented and read by the runtime, and the service that needs it "
Expand Down Expand Up @@ -610,8 +611,10 @@ def pins(text: str) -> List[tuple]:
# container is the failure the textual checks above are reasoning about. Both arms: set, and
# left alone. The unset arm is the one an operator actually boots, and it must show the empty
# default arriving as empty rather than as the literal `${LAYA_MAX_CONCURRENT:-}`.
_KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION")
_arms = [(dict(zip(_KN, ("4", "16384", "reviewed"))), dict(zip(_KN, ("4", "16384", "reviewed")))),
_KN = ("LAYA_MAX_CONCURRENT", "LAYA_MAX_TOKEN_BUDGET", "LAYA_REVISION",
"LAYA_EXTRA_MODELS", "LAYA_IDLE_UNLOAD_SECONDS")
_arms = [(dict(zip(_KN, ("4", "16384", "reviewed", '{"custom":"./model"}', "30"))),
dict(zip(_KN, ("4", "16384", "reviewed", '{"custom":"./model"}', "30")))),
({}, dict.fromkeys(_KN, ""))]
for _set, _expect in _arms:
proc = subprocess.run(["docker", "compose", "-f", "compose.yaml", "-f", "compose.http.yaml",
Expand Down
Loading