From 38db478dd12bf06499d4e6a99912362c77f1be53 Mon Sep 17 00:00:00 2001 From: AliAA Date: Mon, 28 Sep 2026 08:22:35 +0330 Subject: [PATCH 1/3] fix(agent): serialize inference during OOM fallback --- laya/agent.py | 93 +++++++++++++++++++++---------------- tests/test_runtime_fixes.py | 72 ++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 41 deletions(-) diff --git a/laya/agent.py b/laya/agent.py index 5ca3861c4..8859b1ad3 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -148,7 +148,7 @@ def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str, # The per-inference CPU fallback rewrites shared runtime state (device, dtype, amp) and moves the # model while other threads may be running their own forward, so the demotion and the restore are # serialised. A second request that hits OOM waits here and re-demotes only if it needs to. -_OOM_FALLBACK_LOCK = threading.Lock() +_OOM_FALLBACK_LOCK = threading.RLock() def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any: @@ -264,6 +264,10 @@ class Agent(HookRegistry): hooks_concurrent = True hooks_timeout = None _hooks_lock = None + # A fallback moves the shared model between devices. Instances built without __init__ use the + # fallback lock in _infer; normal Agents receive a private lock so unrelated checkpoints do + # not serialize their forwards. + _infer_lock = None model_id = None # Autocast is chosen per device in __init__; this default covers instances built # without it (for example a hand-constructed runtime in tests). @@ -330,6 +334,7 @@ def __init__( self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout) self._hooks_lock = threading.RLock() if not hooks_concurrent else None self._hooks_mutex = threading.Lock() + self._infer_lock = threading.RLock() self.model_id = model_id_or_path from safetensors.torch import load_file @@ -819,47 +824,53 @@ def run(): b["qtype"].to(self.device), ) - try: - return run() - except (RuntimeError, torch.cuda.OutOfMemoryError) as e: - low = str(e).lower() - # An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare - # "cuda" substring used to be accepted too, so any CUDA-shaped RuntimeError (a - # shape, assert or kernel error) silently and permanently demoted the agent. - if self.device.type != "cpu" and (isinstance(e, torch.cuda.OutOfMemoryError) or "memory" in low): - print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...") - # Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move - # the model for the life of the process, so one oversized request left every - # later call ~10-15x slower on CPU. Demote under the lock, answer this request - # on CPU, then put the runtime back the way it was. - with _OOM_FALLBACK_LOCK: - # Recorded on entry, under the same lock as the demotion: the count and - # reason must be readable by /health without racing a concurrent fallback. - self.cpu_fallback_count += 1 - self.last_fallback_reason = str(e) - held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled - had_fast = self._fast is not None - # FastLaya keeps copied CUDA weights and replaces model.forward. Move - # the model first without that replacement, or the retry would still - # execute on the failed CUDA fast path. - self.deaccelerate() - self.device = torch.device("cpu") - self.dtype = torch.float32 - self.amp_enabled = False - self.model.to(self.device) - if torch.cuda.is_available(): - torch.cuda.empty_cache() - try: - return run() - finally: - self._restore_runtime(held_device, held_dtype, held_amp, had_fast) - if use_amp and self.device.type in ("mps", "cpu"): - # Not every MPS/CPU build implements autocast for every op. Drop to full - # precision once rather than failing the request. - self.amp_enabled = False - self.dtype = torch.float32 + # Hold the instance lock across the first forward and any CPU retry. Otherwise another + # thread can prepare CUDA inputs, then observe this request moving the shared model to CPU + # and fail with a device-mismatch error. The fallback lock remains separate so its + # observability and cross-agent transition bookkeeping stay serialized as before. + infer_lock = getattr(self, "_infer_lock", None) or _OOM_FALLBACK_LOCK + with infer_lock: + try: return run() - raise + except (RuntimeError, torch.cuda.OutOfMemoryError) as e: + low = str(e).lower() + # An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare + # "cuda" substring used to be accepted too, so any CUDA-shaped RuntimeError (a + # shape, assert or kernel error) silently and permanently demoted the agent. + if self.device.type != "cpu" and (isinstance(e, torch.cuda.OutOfMemoryError) or "memory" in low): + print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...") + # Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move + # the model for the life of the process, so one oversized request left every + # later call ~10-15x slower on CPU. Demote under the lock, answer this request + # on CPU, then put the runtime back the way it was. + with _OOM_FALLBACK_LOCK: + # Recorded on entry, under the same lock as the demotion: the count and + # reason must be readable by /health without racing a concurrent fallback. + self.cpu_fallback_count += 1 + self.last_fallback_reason = str(e) + held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled + had_fast = self._fast is not None + # FastLaya keeps copied CUDA weights and replaces model.forward. Move + # the model first without that replacement, or the retry would still + # execute on the failed CUDA fast path. + self.deaccelerate() + self.device = torch.device("cpu") + self.dtype = torch.float32 + self.amp_enabled = False + self.model.to(self.device) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + try: + return run() + finally: + self._restore_runtime(held_device, held_dtype, held_amp, had_fast) + if use_amp and self.device.type in ("mps", "cpu"): + # Not every MPS/CPU build implements autocast for every op. Drop to full + # precision once rather than failing the request. + self.amp_enabled = False + self.dtype = torch.float32 + return run() + raise def _forward(self, b: Dict): """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs.""" diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py index a89e08906..139fbd55a 100644 --- a/tests/test_runtime_fixes.py +++ b/tests/test_runtime_fixes.py @@ -7,6 +7,9 @@ """ import os import sys +import concurrent.futures +import threading +import time from contextlib import nullcontext from types import SimpleNamespace @@ -290,6 +293,75 @@ def __call__(self, *args): oom._infer(oom_batch) # a second OOM accumulates check("oom-fallback/second OOM counts too", oom.cpu_fallback_count, 2) + +class ConcurrentOOMModel(ImmovableModel): + """Make an in-flight GPU call fail if another call reaches the shared model.""" + + def __init__(self): + self.active = 0 + self.guard = threading.Lock() + self.first_gpu_started = threading.Event() + self.release_first = threading.Event() + self.placed = "cuda" + + def __call__(self, *args): + with self.guard: + self.active += 1 + concurrent_call = self.active > 1 and self.placed == "cuda" + first_gpu_call = self.placed == "cuda" and not self.first_gpu_started.is_set() + if first_gpu_call: + self.first_gpu_started.set() + self.release_first.wait(2) + try: + if concurrent_call: + raise RuntimeError("Expected all tensors to be on the same device, cuda:0 and cpu") + if self.placed == "cuda": + raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB") + return torch.zeros((1, 2)), torch.zeros((1, 2)) + finally: + with self.guard: + self.active -= 1 + + +race_model = ConcurrentOOMModel() +race_agent = _bare_agent(race_model, dtype=torch.float16) +race_agent.device = torch.device("cuda") +race_agent._infer_lock = threading.RLock() + + +class DeviceAgnosticInput: + shape = (1, 8) + + def to(self, device): + return self + + +race_batch = {key: DeviceAgnosticInput() for key in batch} + + +def run_race_request(): + # Set an independent ready flag before entering _infer so the second worker is definitely + # submitted while the first worker is still inside the shared model. + second_ready.set() + return race_agent._infer(race_batch) + + +second_ready = threading.Event() +with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(race_agent._infer, race_batch) + check("oom-race/first GPU call started", race_model.first_gpu_started.wait(1), True) + second = pool.submit(run_race_request) + check("oom-race/second request submitted", second_ready.wait(1), True) + time.sleep(0.05) + race_model.release_first.set() + try: + first.result(timeout=3) + second.result(timeout=3) + race_ok = True + except Exception: + race_ok = False +check("oom-race/concurrent requests stay on matching devices", race_ok, True) + # a plain forward never touches the counters check("oom-fallback/plain CPU infer stays 0", cpu_disabled.cpu_fallback_count, 0) check("oom-fallback/plain CPU reason stays None", cpu_disabled.last_fallback_reason, None) From 6b7941265c53a03e5ae8dafdad41ad4bec7597e0 Mon Sep 17 00:00:00 2001 From: AliAA Date: Wed, 30 Sep 2026 08:24:17 +0330 Subject: [PATCH 2/3] fix(agent): preserve concurrent forwards during OOM fallback --- laya/agent.py | 102 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 22 deletions(-) diff --git a/laya/agent.py b/laya/agent.py index 8859b1ad3..173bfdf57 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -150,6 +150,57 @@ def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str, # serialised. A second request that hits OOM waits here and re-demotes only if it needs to. _OOM_FALLBACK_LOCK = threading.RLock() +class _InferenceRWLock: + """Writer-preferred lock for concurrent forwards and exclusive device fallback.""" + + def __init__(self): + self._condition = threading.Condition(threading.Lock()) + self._readers = 0 + self._writer = False + self._writers_waiting = 0 + + def acquire_read(self): + with self._condition: + while self._writer or self._writers_waiting: + self._condition.wait() + self._readers += 1 + + def release_read(self): + with self._condition: + self._readers -= 1 + if not self._readers: + self._condition.notify_all() + + def acquire_write(self): + with self._condition: + self._writers_waiting += 1 + try: + while self._writer or self._readers: + self._condition.wait() + self._writer = True + finally: + self._writers_waiting -= 1 + + def release_write(self): + with self._condition: + self._writer = False + self._condition.notify_all() + + def __enter__(self): + self.acquire_read() + return self + + def __exit__(self, exc_type, exc, tb): + self.release_read() + + def upgrade_to_write(self): + self.release_read() + self.acquire_write() + + def downgrade_to_read(self): + self.release_write() + self.acquire_read() + def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any: """Tokenizer for a checkpoint, parsed once per process. @@ -334,7 +385,7 @@ def __init__( self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout) self._hooks_lock = threading.RLock() if not hooks_concurrent else None self._hooks_mutex = threading.Lock() - self._infer_lock = threading.RLock() + self._infer_lock = _InferenceRWLock() self.model_id = model_id_or_path from safetensors.torch import load_file @@ -829,6 +880,7 @@ def run(): # and fail with a device-mismatch error. The fallback lock remains separate so its # observability and cross-agent transition bookkeeping stay serialized as before. infer_lock = getattr(self, "_infer_lock", None) or _OOM_FALLBACK_LOCK + rw_upgrade = hasattr(infer_lock, "upgrade_to_write") with infer_lock: try: return run() @@ -843,27 +895,33 @@ def run(): # the model for the life of the process, so one oversized request left every # later call ~10-15x slower on CPU. Demote under the lock, answer this request # on CPU, then put the runtime back the way it was. - with _OOM_FALLBACK_LOCK: - # Recorded on entry, under the same lock as the demotion: the count and - # reason must be readable by /health without racing a concurrent fallback. - self.cpu_fallback_count += 1 - self.last_fallback_reason = str(e) - held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled - had_fast = self._fast is not None - # FastLaya keeps copied CUDA weights and replaces model.forward. Move - # the model first without that replacement, or the retry would still - # execute on the failed CUDA fast path. - self.deaccelerate() - self.device = torch.device("cpu") - self.dtype = torch.float32 - self.amp_enabled = False - self.model.to(self.device) - if torch.cuda.is_available(): - torch.cuda.empty_cache() - try: - return run() - finally: - self._restore_runtime(held_device, held_dtype, held_amp, had_fast) + if rw_upgrade: + infer_lock.upgrade_to_write() + try: + with _OOM_FALLBACK_LOCK: + # Recorded on entry, under the same lock as the demotion: the count and + # reason must be readable by /health without racing a concurrent fallback. + self.cpu_fallback_count += 1 + self.last_fallback_reason = str(e) + held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled + had_fast = self._fast is not None + # FastLaya keeps copied CUDA weights and replaces model.forward. Move + # the model first without that replacement, or the retry would still + # execute on the failed CUDA fast path. + self.deaccelerate() + self.device = torch.device("cpu") + self.dtype = torch.float32 + self.amp_enabled = False + self.model.to(self.device) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + try: + return run() + finally: + self._restore_runtime(held_device, held_dtype, held_amp, had_fast) + finally: + if rw_upgrade: + infer_lock.downgrade_to_read() if use_amp and self.device.type in ("mps", "cpu"): # Not every MPS/CPU build implements autocast for every op. Drop to full # precision once rather than failing the request. From b4e9414c45ff74d1096ffbce9108c09729f92d9f Mon Sep 17 00:00:00 2001 From: AliAA Date: Wed, 30 Sep 2026 08:24:26 +0330 Subject: [PATCH 3/3] test(agent): cover concurrent OOM fallback --- laya/agent.py | 3 ++- tests/test_runtime_fixes.py | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/laya/agent.py b/laya/agent.py index 173bfdf57..25ecead3e 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -875,7 +875,8 @@ def run(): b["qtype"].to(self.device), ) - # Hold the instance lock across the first forward and any CPU retry. Otherwise another + # Hold a shared read lock across normal forwards. If a forward OOMs, it upgrades to an exclusive + # lock before moving the shared model; otherwise another # thread can prepare CUDA inputs, then observe this request moving the shared model to CPU # and fail with a device-mismatch error. The fallback lock remains separate so its # observability and cross-agent transition bookkeeping stay serialized as before. diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py index 139fbd55a..2c6630bb2 100644 --- a/tests/test_runtime_fixes.py +++ b/tests/test_runtime_fixes.py @@ -18,7 +18,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows # noqa: E402 +from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _InferenceRWLock, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows # noqa: E402 from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402 PASS, FAIL = [], [] @@ -295,7 +295,7 @@ def __call__(self, *args): class ConcurrentOOMModel(ImmovableModel): - """Make an in-flight GPU call fail if another call reaches the shared model.""" + """Make concurrent GPU calls hit OOM so fallback serialization can be exercised.""" def __init__(self): self.active = 0 @@ -313,8 +313,6 @@ def __call__(self, *args): self.first_gpu_started.set() self.release_first.wait(2) try: - if concurrent_call: - raise RuntimeError("Expected all tensors to be on the same device, cuda:0 and cpu") if self.placed == "cuda": raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB") return torch.zeros((1, 2)), torch.zeros((1, 2)) @@ -326,7 +324,7 @@ def __call__(self, *args): race_model = ConcurrentOOMModel() race_agent = _bare_agent(race_model, dtype=torch.float16) race_agent.device = torch.device("cuda") -race_agent._infer_lock = threading.RLock() +race_agent._infer_lock = _InferenceRWLock() class DeviceAgnosticInput: