Skip to content
Prev Previous commit
Next Next commit
feat: add async client with configurable retry policy
AsyncAgent wraps Agent and runs the forward pass in a thread executor. Laya
inference is local compute rather than network I/O, so the point is keeping
the event loop responsive and letting callers fan out with asyncio.gather,
not overlapping waits. batch() evaluates many states under a concurrency limit.

RetryPolicy provides exponential backoff with jitter. It retries only
InferenceError by default: question and configuration errors are deterministic,
so retrying them would just fail identically while hiding the real cause.
  • Loading branch information
teja teja
teja authored and teja committed Sep 18, 2026
commit 34cafcee1ca5e873c23d03cd1a8d8622e3782c7a
153 changes: 153 additions & 0 deletions laya/aio.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
"""Async client and retry policy for laya.

Laya inference is local CPU/GPU compute, not a network call, so there is no I/O to await.
:class:`AsyncAgent` therefore runs the forward pass in a thread executor: it keeps the
event loop responsive and lets callers fan out with ``asyncio.gather`` without blocking.
"""
import asyncio
import functools
import logging
import random
import time
from dataclasses import dataclass
from typing import Any, Dict, Optional, Sequence, Tuple, Type, Union

from .agent import Agent
from .errors import InferenceError, LayaError

logger = logging.getLogger("laya")

__all__ = ["RetryPolicy", "AsyncAgent", "with_retries"]


@dataclass(frozen=True)
class RetryPolicy:
"""Exponential backoff with jitter.

Only transient failures are retried. Question/config errors are deterministic -- a
retry would fail identically -- so by default only :class:`InferenceError` is retried.
"""

max_attempts: int = 3
initial_backoff: float = 0.1
max_backoff: float = 5.0
multiplier: float = 2.0
jitter: bool = True
retry_on: Tuple[Type[BaseException], ...] = (InferenceError,)

def __post_init__(self):
if self.max_attempts < 1:
raise ValueError("max_attempts must be >= 1, got %r" % self.max_attempts)
if self.initial_backoff < 0 or self.max_backoff < 0:
raise ValueError("backoff values must be non-negative")

def should_retry(self, exc: BaseException, attempt: int) -> bool:
return attempt < self.max_attempts and isinstance(exc, self.retry_on)

def backoff_for(self, attempt: int) -> float:
"""Delay in seconds before ``attempt`` + 1 (attempt is 1-based)."""
delay = min(self.max_backoff, self.initial_backoff * (self.multiplier ** (attempt - 1)))
if self.jitter:
delay *= 0.5 + random.random() / 2.0
return delay


NO_RETRY = RetryPolicy(max_attempts=1)


def with_retries(fn, policy: RetryPolicy, *args, **kwargs):
"""Call ``fn`` under a retry policy (synchronous)."""
attempt = 0
while True:
attempt += 1
try:
return fn(*args, **kwargs)
except Exception as exc:
if not policy.should_retry(exc, attempt):
raise
delay = policy.backoff_for(attempt)
logger.warning(
"laya: attempt %d/%d failed (%s: %s); retrying in %.2fs",
attempt, policy.max_attempts, type(exc).__name__, exc, delay,
)
time.sleep(delay)


async def _await_retries(fn, policy: RetryPolicy, *args, **kwargs):
attempt = 0
loop = asyncio.get_event_loop()
while True:
attempt += 1
try:
return await loop.run_in_executor(None, functools.partial(fn, *args, **kwargs))
except Exception as exc:
if not policy.should_retry(exc, attempt):
raise
delay = policy.backoff_for(attempt)
logger.warning(
"laya: attempt %d/%d failed (%s: %s); retrying in %.2fs",
attempt, policy.max_attempts, type(exc).__name__, exc, delay,
)
await asyncio.sleep(delay)


class AsyncAgent:
"""Async wrapper around :class:`laya.Agent`.

Example::

agent = await AsyncAgent.load("convaiinnovations/laya")
result = await agent.system_one(state, questions)
batch = await agent.batch([state_a, state_b], questions)
"""

def __init__(self, agent: Agent, retry: Optional[RetryPolicy] = None):
self._agent = agent
self.retry = retry or RetryPolicy()

@classmethod
async def load(
cls,
model_id_or_path: str = "convaiinnovations/laya",
retry: Optional[RetryPolicy] = None,
**kwargs,
) -> "AsyncAgent":
"""Load the model off the event loop."""
loop = asyncio.get_event_loop()
agent = await loop.run_in_executor(
None, functools.partial(Agent, model_id_or_path, **kwargs)
)
return cls(agent, retry=retry)

@property
def agent(self) -> Agent:
"""The wrapped synchronous agent."""
return self._agent

async def system_one(self, state, questions: Dict[str, Dict[str, Any]], retry=None):
"""Evaluate typed questions without blocking the event loop."""
return await _await_retries(
self._agent.system_one, retry or self.retry, state, questions
)

predict = system_one

async def batch(
self,
states: Sequence[Any],
questions: Dict[str, Dict[str, Any]],
max_concurrency: int = 8,
return_exceptions: bool = False,
):
"""Evaluate the same questions over many states concurrently."""
if max_concurrency < 1:
raise ValueError("max_concurrency must be >= 1, got %r" % max_concurrency)
sem = asyncio.Semaphore(max_concurrency)

async def one(st):
async with sem:
return await self.system_one(st, questions)

return await asyncio.gather(
*(one(s) for s in states), return_exceptions=return_exceptions
)
149 changes: 149 additions & 0 deletions tests/test_aio.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
"""P1: async client and retry policy."""
import asyncio

import pytest

from conftest import FakeAgent, noul
from laya.aio import AsyncAgent, RetryPolicy, with_retries
from laya.errors import InferenceError, QuestionError

NOUL_Q = {"type": "noul", "instructions": "is it spam?"}


def run(coro):
return asyncio.run(coro)


class TestRetryPolicy:
def test_backoff_grows(self):
p = RetryPolicy(initial_backoff=0.1, multiplier=2.0, jitter=False)
assert p.backoff_for(1) == pytest.approx(0.1)
assert p.backoff_for(2) == pytest.approx(0.2)
assert p.backoff_for(3) == pytest.approx(0.4)

def test_backoff_is_capped(self):
p = RetryPolicy(initial_backoff=1.0, multiplier=10.0, max_backoff=2.0, jitter=False)
assert p.backoff_for(5) == pytest.approx(2.0)

def test_jitter_stays_within_bounds(self):
p = RetryPolicy(initial_backoff=1.0, multiplier=1.0, jitter=True)
for _ in range(50):
assert 0.5 <= p.backoff_for(1) <= 1.0

def test_retries_only_listed_types(self):
p = RetryPolicy(max_attempts=3)
assert p.should_retry(InferenceError("x"), 1)
assert not p.should_retry(QuestionError("x"), 1), "deterministic errors must not retry"

def test_stops_at_max_attempts(self):
p = RetryPolicy(max_attempts=2)
assert p.should_retry(InferenceError("x"), 1)
assert not p.should_retry(InferenceError("x"), 2)

def test_rejects_bad_config(self):
with pytest.raises(ValueError):
RetryPolicy(max_attempts=0)


class TestWithRetries:
def test_succeeds_after_transient_failure(self):
calls = []

def flaky():
calls.append(1)
if len(calls) < 3:
raise InferenceError("transient")
return "ok"

policy = RetryPolicy(max_attempts=5, initial_backoff=0.001, jitter=False)
assert with_retries(flaky, policy) == "ok"
assert len(calls) == 3

def test_gives_up_and_reraises(self):
def always():
raise InferenceError("down")

with pytest.raises(InferenceError):
with_retries(always, RetryPolicy(max_attempts=2, initial_backoff=0.001, jitter=False))

def test_does_not_retry_deterministic_errors(self):
calls = []

def bad():
calls.append(1)
raise QuestionError("malformed")

with pytest.raises(QuestionError):
with_retries(bad, RetryPolicy(max_attempts=5, initial_backoff=0.001))
assert len(calls) == 1, "must fail fast"


class TestAsyncAgent:
def test_system_one_awaits(self):
agent = AsyncAgent(FakeAgent(lambda s, q: {"x": noul(0.9)}))
out = run(agent.system_one("state", {"x": NOUL_Q}))
assert out["answers"]["x"].noul == 0.9

def test_batch_covers_every_state(self):
agent = AsyncAgent(FakeAgent(lambda s, q: {"x": noul(len(s) / 10)}))
out = run(agent.batch(["a", "bb", "ccc"], {"x": NOUL_Q}))
assert [r["answers"]["x"].noul for r in out] == [0.1, 0.2, 0.3]

def test_batch_respects_concurrency_limit(self):
import threading
live, peak, lock = [0], [0], threading.Lock()

def slow(s, q):
with lock:
live[0] += 1
peak[0] = max(peak[0], live[0])
import time as t; t.sleep(0.01)
with lock:
live[0] -= 1
return {"x": noul(0.5)}

agent = AsyncAgent(FakeAgent(slow))
run(agent.batch(["s"] * 12, {"x": NOUL_Q}, max_concurrency=3))
assert peak[0] <= 3

def test_batch_can_return_exceptions(self):
def boom(s, q):
if s == "bad":
raise QuestionError("nope")
return {"x": noul(0.5)}

agent = AsyncAgent(FakeAgent(boom))
out = run(agent.batch(["ok", "bad"], {"x": NOUL_Q}, return_exceptions=True))
assert isinstance(out[1], QuestionError)

def test_batch_rejects_bad_concurrency(self):
agent = AsyncAgent(FakeAgent())
with pytest.raises(ValueError):
run(agent.batch(["a"], {"x": NOUL_Q}, max_concurrency=0))

def test_retries_through_async_path(self):
calls = []

def flaky(s, q):
calls.append(1)
if len(calls) < 2:
raise InferenceError("transient")
return {"x": noul(0.7)}

agent = AsyncAgent(
FakeAgent(flaky), retry=RetryPolicy(max_attempts=3, initial_backoff=0.001, jitter=False)
)
out = run(agent.system_one("s", {"x": NOUL_Q}))
assert out["answers"]["x"].noul == 0.7 and len(calls) == 2

def test_exposes_wrapped_agent(self):
inner = FakeAgent()
assert AsyncAgent(inner).agent is inner

def test_concurrent_calls_do_not_block_each_other(self):
agent = AsyncAgent(FakeAgent(lambda s, q: {"x": noul(0.5)}))

async def main():
return await asyncio.gather(*(agent.system_one(str(i), {"x": NOUL_Q}) for i in range(5)))

assert len(run(main())) == 5