Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions laya/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -595,13 +595,16 @@ def __init__(
if device is not None:
target_device = torch.device(device)
if target_device.type == "cuda" and not torch.cuda.is_available():
print("Warning: CUDA requested but not available. Falling back to CPU.")
warnings.warn("Warning: CUDA requested but not available. Falling back to CPU.",
RuntimeWarning)
self.device = torch.device("cpu")
elif target_device.type == "mps" and not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()):
print("Warning: MPS requested but not available. Falling back to CPU.")
warnings.warn("Warning: MPS requested but not available. Falling back to CPU.",
RuntimeWarning)
self.device = torch.device("cpu")
elif target_device.type == "xpu" and not (hasattr(torch, "xpu") and torch.xpu.is_available()):
print("Warning: XPU requested but not available. Falling back to CPU.")
warnings.warn("Warning: XPU requested but not available. Falling back to CPU.",
RuntimeWarning)
self.device = torch.device("cpu")
else:
self.device = target_device
Expand Down Expand Up @@ -741,15 +744,15 @@ def __init__(
self.accelerate()

if fell_back_from is not None:
print(
warnings.warn(
"\n[laya] Warning: could not place the model on %s, so it is running on CPU.\n"
" Reason: %s\n"
" Inference will be roughly 10-15x slower (~200-500 ms rather than ~35 ms).\n"
" If this is a newer NVIDIA GPU (Blackwell / RTX 50-series), your PyTorch build\n"
" may not support its CUDA architecture:\n"
" pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128\n"
" See https://pytorch.org/get-started/locally/\n"
% (fell_back_from, fell_back_why), flush=True)
% (fell_back_from, fell_back_why), RuntimeWarning)

def accelerate(self, use_graphs: bool = True, strict: bool = False):
"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,
Expand Down Expand Up @@ -779,7 +782,8 @@ def accelerate(self, use_graphs: bool = True, strict: bool = False):
if self._fast is None:
if strict:
raise last
print("Warning: laya fast path unavailable (%s); using the stock forward." % last)
warnings.warn("Warning: laya fast path unavailable (%s); using the stock forward." % last,
RuntimeWarning)
return False
self._stock_forward = self.model.forward
self.model.forward = self._fast.forward
Expand Down Expand Up @@ -838,8 +842,8 @@ def _restore_runtime(self, device, dtype, amp_enabled: bool, had_fast: bool) ->
self.model.to(torch.device("cpu"))
if torch.cuda.is_available():
torch.cuda.empty_cache()
print("Warning: could not move the model back to %s after the CPU retry (%s); "
"staying on CPU." % (device, e))
warnings.warn("Warning: could not move the model back to %s after the CPU retry (%s); "
"staying on CPU." % (device, e), RuntimeWarning)
return
self.device, self.dtype, self.amp_enabled = device, dtype, amp_enabled
if had_fast:
Expand Down Expand Up @@ -1118,7 +1122,8 @@ def run(enabled=None):
# "cuda" substring used to be accepted too, so any CUDA-shaped RuntimeError (a
# shape, assert or kernel error) silently and permanently demoted the agent.
if self.device.type != "cpu" and (isinstance(e, torch.cuda.OutOfMemoryError) or "memory" in low):
print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...")
warnings.warn("Warning: GPU memory exceeded during inference. "
"Retrying this request on CPU...", RuntimeWarning)
# Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move
# the model for the life of the process, so one oversized request left every
# later call ~10-15x slower on CPU. Demote under the write lock (#649) so any
Expand Down Expand Up @@ -1162,8 +1167,9 @@ def run(enabled=None):
finally:
self._amp_failures += 1
if self._amp_failures >= _AMP_FAIL_LIMIT:
print("Warning: autocast failed %d times in a row. Disabling mixed precision."
% self._amp_failures)
warnings.warn("Warning: autocast failed %d times in a row. "
"Disabling mixed precision." % self._amp_failures,
RuntimeWarning)
self.amp_enabled = False
self.dtype = torch.float32
raise
Expand Down
24 changes: 24 additions & 0 deletions tests/test_load_errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,16 @@

Run: python tests/test_load_errors.py
"""
import io
import json
import os
import shutil
import sys
import tempfile
import warnings
from contextlib import redirect_stdout
from pathlib import Path
from unittest.mock import patch

os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_TORCH", "1")
Expand Down Expand Up @@ -220,6 +224,26 @@ def load_error(path, **kw):
check_true("short temperature/says the shape", "list of 3" in str(err), str(err))


# ------------------------------------------- 7. a device fallback warns instead of printing
# Asking for a device this machine does not have falls back to CPU with a note. That note used
# to be a bare `print`, so it landed on stdout -- where `laya --json` and `laya --batch --json`
# write their records and where the MCP server speaks JSON-RPC. A warning goes to stderr and a
# caller can filter it; a print breaks the stream and cannot be silenced.
_stdout = io.StringIO()
with warnings.catch_warnings(record=True) as caught, \
patch("torch.cuda.is_available", return_value=False), \
redirect_stdout(_stdout):
warnings.simplefilter("always")
_fallback = load(str(REPO), device="cuda")
check("device fallback/lands on CPU", _fallback.device.type, "cpu")
check("device fallback/writes nothing to stdout", _stdout.getvalue(), "")
check_true("device fallback/warns instead",
any(issubclass(w.category, RuntimeWarning) and "CUDA requested" in str(w.message)
for w in caught),
str([str(w.message) for w in caught]))
del _fallback


TMP.cleanup()

print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
Expand Down
Loading