Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 27 additions & 10 deletions laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ def _amp_context(device, dtype, enabled: bool):


MPS_AMP_MIN_ROWS_DEFAULT = 5
# An unsupported autocast op fails the same way on every request. Retry that request in
# full precision, and only turn AMP off after this many failures in a row (#351).
_AMP_FAIL_LIMIT = 3


def _mps_amp_min_rows() -> int:
Expand Down Expand Up @@ -269,6 +272,7 @@ class Agent(HookRegistry):
# without it (for example a hand-constructed runtime in tests).
amp_enabled = False
mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT
_amp_failures = 0
# The stock forward is also used by lightweight runtimes built with __new__ in tests.
_fast = None
# Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired
Expand Down Expand Up @@ -805,10 +809,12 @@ def _infer(self, b: Dict):
% (self._fast.max_len, b["input_ids"].shape[1], self._fast.max_len)
)

def run():
# Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes
# effect on the retry. A disabled gate never enters torch.autocast at all.
enabled = self._amp_enabled_for(b["input_ids"].shape[0])
def run(enabled=None):
# Recomputed inside run() so a fallback that moves to CPU takes effect on the
# retry. A disabled gate never enters torch.autocast at all. `enabled=False`
# retries one request in full precision without reading the agent's AMP flag.
if enabled is None:
enabled = self._amp_enabled_for(b["input_ids"].shape[0])
dims = independent_dims() if self._compiled else nullcontext()
with _amp_context(self.device, self.dtype, enabled), dims:
return self.model(
Expand All @@ -820,7 +826,7 @@ def run():
)

try:
return run()
out = run()
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
low = str(e).lower()
# An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare
Expand Down Expand Up @@ -854,12 +860,23 @@ def run():
finally:
self._restore_runtime(held_device, held_dtype, held_amp, had_fast)
if use_amp and self.device.type in ("mps", "cpu"):
# Not every MPS/CPU build implements autocast for every op. Drop to full
# precision once rather than failing the request.
self.amp_enabled = False
self.dtype = torch.float32
return run()
# Not every MPS/CPU build implements autocast for every op. Retry this
# request in full precision. One miss must not turn AMP off; a build that
# lacks the op fails the same way every time, so after a short streak the
# process drops to full precision instead of paying for two forwards (#351).
try:
return run(enabled=False)
finally:
self._amp_failures += 1
if self._amp_failures >= _AMP_FAIL_LIMIT:
print("Warning: autocast failed %d times in a row. Disabling mixed precision."
% self._amp_failures)
self.amp_enabled = False
self.dtype = torch.float32
raise
else:
self._amp_failures = 0
return out

def _forward(self, b: Dict):
"""Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
Expand Down
38 changes: 38 additions & 0 deletions tests/test_portability.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,44 @@ def _to_that_ignores_mps(self, *args, **kwargs):
except Exception as e: # noqa: BLE001
FAIL.append("fallback/restore-failure not survived: %s: %s" % (type(e).__name__, e))

# one unsupported autocast op retries that request and leaves AMP on. Three misses in a
# row disable it, so a build without the op does not pay for two forwards forever (#351).
# CPU so the MPS row gate does not hide the branch.
class _AutocastMisses(torch.nn.Module):
def __init__(self):
super().__init__()
self.dummy = torch.nn.Parameter(torch.zeros(1))
self.calls = 0

def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):
self.calls += 1
if self.calls in (1, 3, 5):
raise RuntimeError("User specified an unsupported autocast device_type cpu")
logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1]))
logits[:, 0] = 1.0
return logits, torch.tensor([[1.0, 0.0]])

agent = _bare_agent(_AutocastMisses())
agent.device = torch.device("cpu")
agent.amp_enabled = True
agent.dtype = torch.float16
try:
result = agent.predict({"body": "some state"}, QUESTIONS)
check("autocast/answers after one miss", result["answers"]["q"]["choice"], "a")
check("autocast/one miss retries once", agent.model.calls, 2)
check("autocast/one miss keeps amp", agent.amp_enabled, True)
check("autocast/one miss keeps dtype", agent.dtype, torch.float16)
agent.predict({"body": "another state"}, QUESTIONS)
check("autocast/two misses keep amp", agent.amp_enabled, True)
agent.predict({"body": "third state"}, QUESTIONS)
check("autocast/third miss disables amp", agent.amp_enabled, False)
check("autocast/third miss drops dtype", agent.dtype, torch.float32)
check("autocast/three misses are six forwards", agent.model.calls, 6)
agent.predict({"body": "later state"}, QUESTIONS)
check("autocast/later request is one forward", agent.model.calls, 7)
except Exception as e: # noqa: BLE001
FAIL.append("autocast/streak was not survived: %s: %s" % (type(e).__name__, e))


print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
Expand Down
34 changes: 32 additions & 2 deletions tests/test_runtime_fixes.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,11 @@ def forward(self, input_ids, attention_mask):


class Flaky:
"""Fail the autocast attempt of the first three requests; the fp32 retry succeeds."""

def __call__(self, *args):
calls["n"] += 1
if calls["n"] == 1:
if calls["n"] in (1, 3, 5):
raise RuntimeError("autocast not supported on this build")
return torch.zeros((1, 2)), torch.zeros((1, 2))

Expand All @@ -162,8 +164,36 @@ def __call__(self, *args):
}
flaky = _bare_agent(Flaky(), dtype=torch.bfloat16, amp=True)
flaky._infer(batch)
check("infer/falls back and disables amp", flaky.amp_enabled, False)
check("infer/one miss keeps amp", flaky.amp_enabled, True)
check("infer/one miss keeps dtype", flaky.dtype, torch.bfloat16)
check("infer/retried once", calls["n"], 2)
flaky._infer(batch)
check("infer/two misses keep amp", flaky.amp_enabled, True)
flaky._infer(batch)
check("infer/third miss disables amp", flaky.amp_enabled, False)
check("infer/third miss drops dtype", flaky.dtype, torch.float32)
check("infer/three misses retried", calls["n"], 6)
flaky._infer(batch)
check("infer/later request is one forward", calls["n"], 7)


class OneMiss:
def __init__(self):
self.n = 0

def __call__(self, *args):
self.n += 1
if self.n == 1:
raise RuntimeError("autocast not supported on this build")
return torch.zeros((1, 2)), torch.zeros((1, 2))


streak = _bare_agent(OneMiss(), dtype=torch.bfloat16, amp=True)
streak._infer(batch)
check("infer/streak is one after a miss", streak._amp_failures, 1)
streak._infer(batch)
check("infer/a clean forward clears the streak", streak._amp_failures, 0)
check("infer/a clean forward keeps amp", streak.amp_enabled, True)


class Boom:
Expand Down