From 3b92229b17713b3334359e6bdd37627d21734fad Mon Sep 17 00:00:00 2001 From: thread3d Date: Sat, 24 Oct 2026 08:46:39 +0200 Subject: [PATCH] fix(lang): count letters the interpreter's Unicode database does not know str.isalpha() and unicodedata.category answer from the Unicode database bundled with the host interpreter, and the supported versions disagree: Python 3.10 ships Unicode 13.0, 3.11 14.0, 3.12 15.0, 3.13 15.1. A letter added after that database is category Cn -- *unassigned* -- so isalpha() is False and _script_counts dropped it before its "other" branch could count it, which that branch's own comment exists to prevent. Text written in Kawi, Nag Mundari, Todhri, Vithkuqi, Old Uyghur, CJK extension I, ... then reported script="unknown", i.e. "no letters detected", and Router._route reads that as "use the default checkpoint" rather than as non-Latin text. With Router(default= "english") - the documented setting for a mostly-English deployment - that is a misroute to the checkpoint that collapses off English; and laya.is_english() / analyse() gave different answers for the same input on different supported Pythons. _is_letter() now consults a generated table of letters that Unicode 16.0 assigns and Unicode 14.0 does not (plus the scripts Unicode 14 introduced, so Python 3.10 matches too), gated on the character being Cn here: a character this interpreter already knows is still decided by isalpha() alone. Verified by comparing _script_counts for all 1 114 112 codepoints, old vs new, on three interpreters: Unicode 16.0 -> 0 differences; Unicode 15.1 -> 4302, every one a codepoint 15.1 does not know; Unicode 14.0 -> 9272 of the same kind; and on each, zero codepoints the interpreter knows changed answer. --- laya/lang.py | 100 +++++++++++++++++++++++++++++++++++++-- tests/test_lang_guess.py | 34 +++++++++++++ 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/laya/lang.py b/laya/lang.py index 66c0b48d0..044e8b24b 100644 --- a/laya/lang.py +++ b/laya/lang.py @@ -7,8 +7,10 @@ languages (French 0.487, Spanish 0.480). So the signal that matters most is *script*, and the secondary signal is whether Latin text is English. -Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic and -is explicitly best-effort: pass an explicit model or `lang=` when you already know the language. +Script detection is exact, and answers the same way on every supported interpreter: a letter is +decided from the text, never from the Unicode database the host happens to ship (`_is_letter`). +The Latin-script language guess is a stopword/diacritic heuristic and is explicitly best-effort: +pass an explicit model or `lang=` when you already know the language. """ import re import unicodedata @@ -45,6 +47,98 @@ ("han", ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF))), ] +# Letters the running interpreter's Unicode database may not know (see `_is_letter`). +# +# "Letters assigned after Unicode 14.0, plus the scripts Unicode 14 introduced": every codepoint here +# is category ``L*`` in Unicode 16.0 and ``Cn`` (unassigned) in Unicode 14.0 -- except the Unicode 14 +# scripts themselves, which Unicode 14.0 does know and Unicode 13.0 (Python 3.10) does not, so 3.10 +# gets the same answer. Generated from the two databases, not typed by hand; the CJK extensions are +# whole blocks because every ideograph in them is a letter. +_POST_DATABASE_LETTER_RANGES = ( + (0x01C89, 0x01C8A), # CYRILLIC CAPITAL LETTER TJE + (0x0A7CB, 0x0A7CD), # LATIN CAPITAL LETTER RAMS HORN + (0x0A7DA, 0x0A7DC), # LATIN CAPITAL LETTER LAMBDA + (0x10570, 0x1057A), # VITHKUQI (Unicode 14) + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x105C0, 0x105F3), # TODHRI (Unicode 16) + (0x10D4A, 0x10D65), # GARAY (Unicode 16) + (0x10D6F, 0x10D85), + (0x10EC2, 0x10EC4), # ARABIC additions + (0x10F70, 0x10F81), # OLD UYGHUR (Unicode 14) + (0x1123F, 0x11240), # KHOJKI additions + (0x11380, 0x11389), # TULU-TIGALARI (Unicode 16) + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113B7), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11BC0, 0x11BE0), # SUNUWAR (Unicode 16) + (0x11F02, 0x11F02), # KAWI (Unicode 15) + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x12F90, 0x12FF0), # CYPRO-MINOAN (Unicode 14) + (0x1342F, 0x1342F), # EGYPTIAN HIEROGLYPH additions + (0x13441, 0x13446), + (0x13460, 0x143FA), + (0x16100, 0x1611D), # GURUNG KHEMA (Unicode 16) + (0x16A70, 0x16ABE), # TANGSA (Unicode 14) + (0x16D40, 0x16D6C), # KIRAT RAI (Unicode 16) + (0x18CFF, 0x18CFF), # KHITAN SMALL SCRIPT addition + (0x1B132, 0x1B132), # HIRAGANA / KATAKANA additions + (0x1B155, 0x1B155), + (0x1DF25, 0x1DF2A), # LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK + (0x1E030, 0x1E06D), # MODIFIER LETTER additions + (0x1E290, 0x1E2AD), # TOTO (Unicode 14) + (0x1E4D0, 0x1E4EB), # NAG MUNDARI (Unicode 15) + (0x1E5D0, 0x1E5ED), # OL ONAL (Unicode 16) + (0x1E5F0, 0x1E5F0), + (0x2B739, 0x2B739), # CJK UNIFIED IDEOGRAPH extensions H and I + (0x2EBF0, 0x2EE5D), + (0x31350, 0x323AF), +) +# Cheapest rejection for a character the table could match: everything below the first range start is +# never consulted (the ASCII and Latin text almost every request carries), so the common path stays +# `str.isalpha()` plus one integer compare. +_POST_DATABASE_LETTER_FLOOR = _POST_DATABASE_LETTER_RANGES[0][0] + + +def _is_letter(ch: str) -> bool: + """Whether `ch` is a letter, as the *newest* interpreter would say, not just this one. + + ``str.isalpha()`` and ``unicodedata.category`` are properties of the interpreter's bundled + Unicode database, and the supported versions disagree: Python 3.10 ships Unicode 13.0, 3.11 14.0, + 3.12 15.0. A letter added since then is ``Cn`` -- *unassigned* -- on the older interpreter, so + `isalpha()` is False and it was dropped before `_script_counts`' "other" branch could count it. + Text written in one of those scripts therefore reported ``script="unknown"``, which is "no letters + detected", and `Router._route` reads that as the default checkpoint rather than as non-Latin text + -- the exact failure the "other" branch exists to prevent, and it made `analyse`/`is_english` + answer differently for the same input depending on the interpreter. + + The table only SUPPLIES the answer a newer database would have given: a character this interpreter + already knows is decided by `isalpha()` alone, so behaviour on the newest interpreter is unchanged + codepoint for codepoint. What is a letter never depends on the host any more, for the scripts + listed above. + """ + if ch.isalpha(): + return True + if ord(ch) < _POST_DATABASE_LETTER_FLOOR: + return False + if unicodedata.category(ch) != "Cn": + return False + cp = ord(ch) + for lo, hi in _POST_DATABASE_LETTER_RANGES: + if lo <= cp <= hi: + return True + return False + + # Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is # weighted and a margin is required before calling something non-English. # @@ -329,7 +423,7 @@ def _script_counts(text: str) -> Dict[str, int]: counts: Dict[str, int] = {} latin = 0 for ch in text: - if not ch.isalpha(): + if not _is_letter(ch): continue cp = ord(ch) if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A: diff --git a/tests/test_lang_guess.py b/tests/test_lang_guess.py index 864d647c3..e6b6bc21c 100644 --- a/tests/test_lang_guess.py +++ b/tests/test_lang_guess.py @@ -576,6 +576,40 @@ def _load(name): check("mixed-script/latin-only acronym run is still english", _route.route("MON DES EST LA")["model"], "english") +# Scripts newer than the interpreter's own Unicode database. `str.isalpha()` answers from that +# database, so a letter added after it is "unassigned" and was dropped before `_script_counts`'s +# "other" branch could count it: the state then read as `script="unknown"`, which `Router._route` +# treats as "no letters detected" and answers with the configured default -- the exact failure the +# "other" branch exists to prevent. Python 3.10 ships Unicode 13.0, 3.11 14.0 and 3.12 15.0, so this +# is a supported-version difference in a public answer (`laya.is_english`). `_is_letter` supplies +# the answer a newer database would have given and changes nothing it already knows, which +# `tests/test_lang_guess.py` cannot see: these cases fail on the older interpreters without it. +POST_DATABASE_SCRIPTS = [ + ("𑼁𑼂𑼃", "kawi"), # Unicode 15.0 + ("𞓑𞓒", "nag mundari"), # Unicode 15.0 + ("𐗀𐗁", "todhri"), # Unicode 16.0 + ("𐕱𐕲", "vithkuqi"), # Unicode 14.0 + ("𐽱𐽲", "old uyghur"), # Unicode 14.0 + ("𮯰𮯱", "CJK extension I"), # Unicode 15.1 +] +for _text, _name in POST_DATABASE_SCRIPTS: + check("newer-script/%s is not 'unknown'" % _name, analyse(_text)["script"], "other") + check("newer-script/%s is not english" % _name, analyse(_text)["is_english"], False) + check("newer-script/%s routes multilingual" % _name, + _route.route(_text)["model"], "multilingual") + +# The table only answers for characters this interpreter does not know: a symbol, a digit and a +# mark stay non-letters, so an emoji-only state is still "no letters" and an English request is +# untouched. Checked rather than assumed -- these are the shapes a blanket "anything above the +# floor is a letter" would break. +check("newer-script/emoji are not letters", analyse("\U0001F600\U0001F601")["script"], "unknown") +check("newer-script/emoji still route by default", + _route.route("\U0001F600\U0001F601")["model"], _route.default) +check("newer-script/english is unchanged", + analyse("We were billed twice and want a refund")["script"], "latin") +check("newer-script/english still routes english", + _route.route("We were billed twice and want a refund")["model"], "english") + print("\n%d passed, %d failed" % (len(PASS), len(FAIL))) for f in FAIL: print(" FAIL " + f)