Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 29 additions & 16 deletions laya/lang.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import unicodedata
from collections import Counter
from collections.abc import Mapping
from typing import Dict, List, Optional, Union
from typing import Dict, Iterator, List, Optional, Union

# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
_SCRIPT_RANGES = [
Expand Down Expand Up @@ -375,51 +375,63 @@ def _is_letter(ch: str) -> bool:


def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0,
_paths=None, _path=()) -> List[str]:
_paths=None, _path=()) -> Iterator[str]:
"""Collect the string leaves of a state, so detection sees real content.

Keys are ignored: they are usually English field names. Any mapping counts, not only
`dict` -- a UserDict or mappingproxy has the same string values and used to contribute
nothing, which made `analyse` report no letters and the router fall through to English.
"""
if _depth > 6 or state is None:
return []
return
if isinstance(state, str):
if _paths is not None:
_paths.append(_path)
return [state]
yield state
return
if isinstance(state, (bytes, bytearray)):
try:
text = bytes(state).decode("utf-8")
except UnicodeDecodeError:
return []
return
if _paths is not None:
_paths.append(_path)
return [text]
yield text
return
if isinstance(state, Mapping):
out = []
if _paths is None:
for v in state.values():
out.extend(_iter_text(v, _depth + 1))
yield from _iter_text(v, _depth + 1)
else:
for key, v in state.items():
# Do not stringify arbitrary keys or copy an unbounded field name into a report.
bounded = ((type(key) is str and len(key) <= 128)
or (type(key) is int and key.bit_length() <= 63))
path = _path + (key,) if _path is not None and bounded else None
out.extend(_iter_text(v, _depth + 1, _paths, path))
return out
yield from _iter_text(v, _depth + 1, _paths, path)
return
if isinstance(state, (list, tuple)):
out = []
for i, v in enumerate(state):
path = _path + (i,) if _paths is not None and _path is not None else _path
out.extend(_iter_text(v, _depth + 1, _paths, path))
return out
return []
yield from _iter_text(v, _depth + 1, _paths, path)
return


def state_text(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000) -> str:
"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""
if max_chars <= 0 or state is None:
return ""
if isinstance(state, str):
return state if len(state) <= max_chars else state[:max_chars]
if isinstance(state, (bytes, bytearray)):
try:
text = bytes(state).decode("utf-8")
except UnicodeDecodeError:
return ""
return text if len(text) <= max_chars else text[:max_chars]
if not isinstance(state, (Mapping, list, tuple)):
return ""

parts: List[str] = []
budget = max_chars
for leaf in _iter_text(state):
Expand Down Expand Up @@ -1021,9 +1033,10 @@ def analyse(state: Union[str, bytes, Mapping, list, None], *, explain: bool = Fa
# accuracy on `pt`). The cost is lopsided: English sent to multilingual loses a few points,
# the reverse loses calibration. So a state that would go to English is checked line by line
# and field by field.
leaves = _iter_text(state)
# a single line has no other part to be outvoted by, and was just read whole
if len(leaves) > 1 or any("\n" in leaf for leaf in leaves):
it = iter(_iter_text(state))
first = next(it, None)
if first is not None and ("\n" in first or next(it, None) is not None):
found = _non_english_segment(state, diagnostic=diagnostic)
if found:
lang, mixed = found
Expand Down
4 changes: 4 additions & 0 deletions tests/test_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,10 @@ def check(name, got, want):
check("state_text/nested", "deep" in state_text({"a": {"b": ["deep"]}}), True)
check("state_text/list", "x" in state_text(["x", {"y": "z"}]), True)
check("state_text/none", state_text(None), "")
check("state_text/plain string", state_text("hello world"), "hello world")
check("state_text/plain string budget", state_text("hello world", max_chars=5), "hello")
check("state_text/bytes budget", state_text(b"hello world", max_chars=5), "hello")
check("state_text/budget truncation halts", state_text(["part1", "part2", "part3"], max_chars=5), "part1")
# keys must not drive detection: English keys around Hindi content stay non-English
check("state_text/keys ignored",
analyse({"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"})["is_english"], False)
Expand Down
Loading