diff --git a/laya/agent.py b/laya/agent.py index 5ca3861c4..d113022ab 100644 --- a/laya/agent.py +++ b/laya/agent.py @@ -198,6 +198,9 @@ def _amp_context(device, dtype, enabled: bool): MPS_AMP_MIN_ROWS_DEFAULT = 5 +# An unsupported autocast op fails the same way on every request. Retry that request in +# full precision, and only turn AMP off after this many failures in a row (#351). +_AMP_FAIL_LIMIT = 3 def _mps_amp_min_rows() -> int: @@ -269,6 +272,7 @@ class Agent(HookRegistry): # without it (for example a hand-constructed runtime in tests). amp_enabled = False mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT + _amp_failures = 0 # The stock forward is also used by lightweight runtimes built with __new__ in tests. _fast = None # Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired @@ -805,10 +809,12 @@ def _infer(self, b: Dict): % (self._fast.max_len, b["input_ids"].shape[1], self._fast.max_len) ) - def run(): - # Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes - # effect on the retry. A disabled gate never enters torch.autocast at all. - enabled = self._amp_enabled_for(b["input_ids"].shape[0]) + def run(enabled=None): + # Recomputed inside run() so a fallback that moves to CPU takes effect on the + # retry. A disabled gate never enters torch.autocast at all. `enabled=False` + # retries one request in full precision without reading the agent's AMP flag. + if enabled is None: + enabled = self._amp_enabled_for(b["input_ids"].shape[0]) dims = independent_dims() if self._compiled else nullcontext() with _amp_context(self.device, self.dtype, enabled), dims: return self.model( @@ -820,7 +826,7 @@ def run(): ) try: - return run() + out = run() except (RuntimeError, torch.cuda.OutOfMemoryError) as e: low = str(e).lower() # An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare @@ -854,12 +860,23 @@ def run(): finally: self._restore_runtime(held_device, held_dtype, held_amp, had_fast) if use_amp and self.device.type in ("mps", "cpu"): - # Not every MPS/CPU build implements autocast for every op. Drop to full - # precision once rather than failing the request. - self.amp_enabled = False - self.dtype = torch.float32 - return run() + # Not every MPS/CPU build implements autocast for every op. Retry this + # request in full precision. One miss must not turn AMP off; a build that + # lacks the op fails the same way every time, so after a short streak the + # process drops to full precision instead of paying for two forwards (#351). + try: + return run(enabled=False) + finally: + self._amp_failures += 1 + if self._amp_failures >= _AMP_FAIL_LIMIT: + print("Warning: autocast failed %d times in a row. Disabling mixed precision." + % self._amp_failures) + self.amp_enabled = False + self.dtype = torch.float32 raise + else: + self._amp_failures = 0 + return out def _forward(self, b: Dict): """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs.""" diff --git a/tests/test_portability.py b/tests/test_portability.py index e085e0e3f..7a19a921e 100644 --- a/tests/test_portability.py +++ b/tests/test_portability.py @@ -257,6 +257,44 @@ def _to_that_ignores_mps(self, *args, **kwargs): except Exception as e: # noqa: BLE001 FAIL.append("fallback/restore-failure not survived: %s: %s" % (type(e).__name__, e)) + # one unsupported autocast op retries that request and leaves AMP on. Three misses in a + # row disable it, so a build without the op does not pay for two forwards forever (#351). + # CPU so the MPS row gate does not hide the branch. + class _AutocastMisses(torch.nn.Module): + def __init__(self): + super().__init__() + self.dummy = torch.nn.Parameter(torch.zeros(1)) + self.calls = 0 + + def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype): + self.calls += 1 + if self.calls in (1, 3, 5): + raise RuntimeError("User specified an unsupported autocast device_type cpu") + logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1])) + logits[:, 0] = 1.0 + return logits, torch.tensor([[1.0, 0.0]]) + + agent = _bare_agent(_AutocastMisses()) + agent.device = torch.device("cpu") + agent.amp_enabled = True + agent.dtype = torch.float16 + try: + result = agent.predict({"body": "some state"}, QUESTIONS) + check("autocast/answers after one miss", result["answers"]["q"]["choice"], "a") + check("autocast/one miss retries once", agent.model.calls, 2) + check("autocast/one miss keeps amp", agent.amp_enabled, True) + check("autocast/one miss keeps dtype", agent.dtype, torch.float16) + agent.predict({"body": "another state"}, QUESTIONS) + check("autocast/two misses keep amp", agent.amp_enabled, True) + agent.predict({"body": "third state"}, QUESTIONS) + check("autocast/third miss disables amp", agent.amp_enabled, False) + check("autocast/third miss drops dtype", agent.dtype, torch.float32) + check("autocast/three misses are six forwards", agent.model.calls, 6) + agent.predict({"body": "later state"}, QUESTIONS) + check("autocast/later request is one forward", agent.model.calls, 7) + except Exception as e: # noqa: BLE001 + FAIL.append("autocast/streak was not survived: %s: %s" % (type(e).__name__, e)) + print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) for f in FAIL: diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py index a89e08906..cf3a77d14 100644 --- a/tests/test_runtime_fixes.py +++ b/tests/test_runtime_fixes.py @@ -146,9 +146,11 @@ def forward(self, input_ids, attention_mask): class Flaky: + """Fail the autocast attempt of the first three requests; the fp32 retry succeeds.""" + def __call__(self, *args): calls["n"] += 1 - if calls["n"] == 1: + if calls["n"] in (1, 3, 5): raise RuntimeError("autocast not supported on this build") return torch.zeros((1, 2)), torch.zeros((1, 2)) @@ -162,8 +164,36 @@ def __call__(self, *args): } flaky = _bare_agent(Flaky(), dtype=torch.bfloat16, amp=True) flaky._infer(batch) -check("infer/falls back and disables amp", flaky.amp_enabled, False) +check("infer/one miss keeps amp", flaky.amp_enabled, True) +check("infer/one miss keeps dtype", flaky.dtype, torch.bfloat16) check("infer/retried once", calls["n"], 2) +flaky._infer(batch) +check("infer/two misses keep amp", flaky.amp_enabled, True) +flaky._infer(batch) +check("infer/third miss disables amp", flaky.amp_enabled, False) +check("infer/third miss drops dtype", flaky.dtype, torch.float32) +check("infer/three misses retried", calls["n"], 6) +flaky._infer(batch) +check("infer/later request is one forward", calls["n"], 7) + + +class OneMiss: + def __init__(self): + self.n = 0 + + def __call__(self, *args): + self.n += 1 + if self.n == 1: + raise RuntimeError("autocast not supported on this build") + return torch.zeros((1, 2)), torch.zeros((1, 2)) + + +streak = _bare_agent(OneMiss(), dtype=torch.bfloat16, amp=True) +streak._infer(batch) +check("infer/streak is one after a miss", streak._amp_failures, 1) +streak._infer(batch) +check("infer/a clean forward clears the streak", streak._amp_failures, 0) +check("infer/a clean forward keeps amp", streak.amp_enabled, True) class Boom: