diff --git a/laya/agent.py b/laya/agent.py index 5ca3861c4..25ecead3e 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -148,7 +148,58 @@ def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str, # The per-inference CPU fallback rewrites shared runtime state (device, dtype, amp) and moves the # model while other threads may be running their own forward, so the demotion and the restore are # serialised. A second request that hits OOM waits here and re-demotes only if it needs to. -_OOM_FALLBACK_LOCK = threading.Lock() +_OOM_FALLBACK_LOCK = threading.RLock() + +class _InferenceRWLock: + """Writer-preferred lock for concurrent forwards and exclusive device fallback.""" + + def __init__(self): + self._condition = threading.Condition(threading.Lock()) + self._readers = 0 + self._writer = False + self._writers_waiting = 0 + + def acquire_read(self): + with self._condition: + while self._writer or self._writers_waiting: + self._condition.wait() + self._readers += 1 + + def release_read(self): + with self._condition: + self._readers -= 1 + if not self._readers: + self._condition.notify_all() + + def acquire_write(self): + with self._condition: + self._writers_waiting += 1 + try: + while self._writer or self._readers: + self._condition.wait() + self._writer = True + finally: + self._writers_waiting -= 1 + + def release_write(self): + with self._condition: + self._writer = False + self._condition.notify_all() + + def __enter__(self): + self.acquire_read() + return self + + def __exit__(self, exc_type, exc, tb): + self.release_read() + + def upgrade_to_write(self): + self.release_read() + self.acquire_write() + + def downgrade_to_read(self): + self.release_write() + self.acquire_read() def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any: @@ -264,6 +315,10 @@ class Agent(HookRegistry): hooks_concurrent = True hooks_timeout = None _hooks_lock = None + # A fallback moves the shared model between devices. Instances built without __init__ use the + # fallback lock in _infer; normal Agents receive a private lock so unrelated checkpoints do + # not serialize their forwards. + _infer_lock = None model_id = None # Autocast is chosen per device in __init__; this default covers instances built # without it (for example a hand-constructed runtime in tests). @@ -330,6 +385,7 @@ def __init__( self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout) self._hooks_lock = threading.RLock() if not hooks_concurrent else None self._hooks_mutex = threading.Lock() + self._infer_lock = _InferenceRWLock() self.model_id = model_id_or_path from safetensors.torch import load_file @@ -819,47 +875,61 @@ def run(): b["qtype"].to(self.device), ) - try: - return run() - except (RuntimeError, torch.cuda.OutOfMemoryError) as e: - low = str(e).lower() - # An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare - # "cuda" substring used to be accepted too, so any CUDA-shaped RuntimeError (a - # shape, assert or kernel error) silently and permanently demoted the agent. - if self.device.type != "cpu" and (isinstance(e, torch.cuda.OutOfMemoryError) or "memory" in low): - print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...") - # Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move - # the model for the life of the process, so one oversized request left every - # later call ~10-15x slower on CPU. Demote under the lock, answer this request - # on CPU, then put the runtime back the way it was. - with _OOM_FALLBACK_LOCK: - # Recorded on entry, under the same lock as the demotion: the count and - # reason must be readable by /health without racing a concurrent fallback. - self.cpu_fallback_count += 1 - self.last_fallback_reason = str(e) - held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled - had_fast = self._fast is not None - # FastLaya keeps copied CUDA weights and replaces model.forward. Move - # the model first without that replacement, or the retry would still - # execute on the failed CUDA fast path. - self.deaccelerate() - self.device = torch.device("cpu") - self.dtype = torch.float32 - self.amp_enabled = False - self.model.to(self.device) - if torch.cuda.is_available(): - torch.cuda.empty_cache() + # Hold a shared read lock across normal forwards. If a forward OOMs, it upgrades to an exclusive + # lock before moving the shared model; otherwise another + # thread can prepare CUDA inputs, then observe this request moving the shared model to CPU + # and fail with a device-mismatch error. The fallback lock remains separate so its + # observability and cross-agent transition bookkeeping stay serialized as before. + infer_lock = getattr(self, "_infer_lock", None) or _OOM_FALLBACK_LOCK + rw_upgrade = hasattr(infer_lock, "upgrade_to_write") + with infer_lock: + try: + return run() + except (RuntimeError, torch.cuda.OutOfMemoryError) as e: + low = str(e).lower() + # An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare + # "cuda" substring used to be accepted too, so any CUDA-shaped RuntimeError (a + # shape, assert or kernel error) silently and permanently demoted the agent. + if self.device.type != "cpu" and (isinstance(e, torch.cuda.OutOfMemoryError) or "memory" in low): + print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...") + # Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move + # the model for the life of the process, so one oversized request left every + # later call ~10-15x slower on CPU. Demote under the lock, answer this request + # on CPU, then put the runtime back the way it was. + if rw_upgrade: + infer_lock.upgrade_to_write() try: - return run() + with _OOM_FALLBACK_LOCK: + # Recorded on entry, under the same lock as the demotion: the count and + # reason must be readable by /health without racing a concurrent fallback. + self.cpu_fallback_count += 1 + self.last_fallback_reason = str(e) + held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled + had_fast = self._fast is not None + # FastLaya keeps copied CUDA weights and replaces model.forward. Move + # the model first without that replacement, or the retry would still + # execute on the failed CUDA fast path. + self.deaccelerate() + self.device = torch.device("cpu") + self.dtype = torch.float32 + self.amp_enabled = False + self.model.to(self.device) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + try: + return run() + finally: + self._restore_runtime(held_device, held_dtype, held_amp, had_fast) finally: - self._restore_runtime(held_device, held_dtype, held_amp, had_fast) - if use_amp and self.device.type in ("mps", "cpu"): - # Not every MPS/CPU build implements autocast for every op. Drop to full - # precision once rather than failing the request. - self.amp_enabled = False - self.dtype = torch.float32 - return run() - raise + if rw_upgrade: + infer_lock.downgrade_to_read() + if use_amp and self.device.type in ("mps", "cpu"): + # Not every MPS/CPU build implements autocast for every op. Drop to full + # precision once rather than failing the request. + self.amp_enabled = False + self.dtype = torch.float32 + return run() + raise def _forward(self, b: Dict): """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs.""" diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py index a89e08906..2c6630bb2 100644 --- a/tests/test_runtime_fixes.py +++ b/tests/test_runtime_fixes.py @@ -7,6 +7,9 @@ """ import os import sys +import concurrent.futures +import threading +import time from contextlib import nullcontext from types import SimpleNamespace @@ -15,7 +18,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows # noqa: E402 +from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _InferenceRWLock, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows # noqa: E402 from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402 PASS, FAIL = [], [] @@ -290,6 +293,73 @@ def __call__(self, *args): oom._infer(oom_batch) # a second OOM accumulates check("oom-fallback/second OOM counts too", oom.cpu_fallback_count, 2) + +class ConcurrentOOMModel(ImmovableModel): + """Make concurrent GPU calls hit OOM so fallback serialization can be exercised.""" + + def __init__(self): + self.active = 0 + self.guard = threading.Lock() + self.first_gpu_started = threading.Event() + self.release_first = threading.Event() + self.placed = "cuda" + + def __call__(self, *args): + with self.guard: + self.active += 1 + concurrent_call = self.active > 1 and self.placed == "cuda" + first_gpu_call = self.placed == "cuda" and not self.first_gpu_started.is_set() + if first_gpu_call: + self.first_gpu_started.set() + self.release_first.wait(2) + try: + if self.placed == "cuda": + raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB") + return torch.zeros((1, 2)), torch.zeros((1, 2)) + finally: + with self.guard: + self.active -= 1 + + +race_model = ConcurrentOOMModel() +race_agent = _bare_agent(race_model, dtype=torch.float16) +race_agent.device = torch.device("cuda") +race_agent._infer_lock = _InferenceRWLock() + + +class DeviceAgnosticInput: + shape = (1, 8) + + def to(self, device): + return self + + +race_batch = {key: DeviceAgnosticInput() for key in batch} + + +def run_race_request(): + # Set an independent ready flag before entering _infer so the second worker is definitely + # submitted while the first worker is still inside the shared model. + second_ready.set() + return race_agent._infer(race_batch) + + +second_ready = threading.Event() +with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(race_agent._infer, race_batch) + check("oom-race/first GPU call started", race_model.first_gpu_started.wait(1), True) + second = pool.submit(run_race_request) + check("oom-race/second request submitted", second_ready.wait(1), True) + time.sleep(0.05) + race_model.release_first.set() + try: + first.result(timeout=3) + second.result(timeout=3) + race_ok = True + except Exception: + race_ok = False +check("oom-race/concurrent requests stay on matching devices", race_ok, True) + # a plain forward never touches the counters check("oom-fallback/plain CPU infer stays 0", cpu_disabled.cpu_fallback_count, 0) check("oom-fallback/plain CPU reason stays None", cpu_disabled.last_fallback_reason, None)