diff --git a/laya/lang.py b/laya/lang.py index f8ecb3365..58ea55ea7 100644 --- a/laya/lang.py +++ b/laya/lang.py @@ -809,8 +809,10 @@ def analyse(state: Union[str, bytes, Mapping, list, None]) -> Dict[str, object]: """Full detection result for a state. Returns `script`, `script_profile`, `language` (best effort, may be None), - `is_english`, `non_latin_fraction` and `mixed_segment` (the line or field that made a mostly - English state non-English, else None). + `is_english`, `non_latin_fraction` and `mixed_segment`. `mixed_segment` names the line or field + the segment scan itself singled out, else None: a state flipped non-English by the per-value + read below is reported with `mixed_segment` still None, because that read keeps the offending + value in its own local and never writes it back. String values are what get read. When a state has several of them, one non-English value is enough: joining every value into one window let a long English note fill the 4000 characters, diff --git a/tests/test_router.py b/tests/test_router.py index 6e4efc844..dc4d00bb2 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -9,6 +9,7 @@ from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, clamp_temperature, temp_bucket # noqa: E402 from laya.lang import analyse, detect_script, guess_latin_language, is_english, state_text # noqa: E402 +from laya.lang import _non_english_segment # noqa: E402 from laya.router import ( # noqa: E402 BUNDLE_REPO, DEFAULT_MODELS, @@ -532,6 +533,19 @@ def _ms(state, reps=9): check("mixed/routes multilingual: " + label, _r_lat.route(state).model, "multilingual") check("mixed/reason names the segment", "a line or field reads as 'pt'" in _r_lat.route(_TRACE).reason, True) +# ...but only the segment scan fills it. `_non_english_segment` reads 4,000 characters in all across +# every leaf, so a long English first field can leave a foreign one unread; the per-value read that +# then catches it (#384) keeps the leaf in a local and does not report it. `mixed_segment` is None +# here even though `language` names the field's language -- asserted rather than assumed, because a +# future change could fill it and no caller can tell the two provenances apart otherwise. +_BUDGET_NOTE = "Thanks for reaching out. " * 200 # 5,000 chars, first leaf, all English +_BUDGET_BODY = "WIE SPAET IST ES IN KOELN HEUTE ABEND" +check("mixed/segment budget exhausted before the foreign field", + _non_english_segment({"note": _BUDGET_NOTE, "body": _BUDGET_BODY}), None) +check("mixed/the per-value read still names the language", + analyse({"note": _BUDGET_NOTE, "body": _BUDGET_BODY})["language"], "de") +check("mixed/and leaves mixed_segment None", + analyse({"note": _BUDGET_NOTE, "body": _BUDGET_BODY})["mixed_segment"], None) # English stays English: several English lines, a short foreign sign-off, and code pasted into a request # (`os.path` reads as Portuguese, `round(el, 2)` as Spanish, `np.mean(na)` as Portuguese) for label, state in [