From 79077883f26a4ccd817c9497687175158553082d Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Mon, 28 Sep 2026 11:16:56 +0530 Subject: [PATCH 1/9] fix(lang): send undecided Latin text with evidence of another language to multilingual Latin-script text that names no language and has no non-English letters took the router's `default`, English unless configured (#54, path 1). From four words up latin_profile() scores the text, and a single English function word names it `en`, so text still undecided there has none. When it carries two different words from other languages' lists, or a non-English letter too rare for the diacritic rate, that is evidence it is not English: "streiche alle meine geplanten termine" (`alle`, `meine`). One such word is not evidence: English uses `do`, `las`, `solo` and `van` too ("how do my health benefits work"), and a word seen twice ("apaga la luz de la cocina") is still one word. latin_profile() returns the evidence as `other_evidence`, and analyse() reports such text as not English, so it routes to multilingual whatever `default` is. Without evidence nothing changes: English with no function word ("play recently added music"), text with one list word, and languages with no list here keep `default`. MASSIVE 1.1 scenario, test split, 2,974 utterances per locale, end to end: ro-RO 0.343 -> 0.359 cy-GB 0.114 -> 0.128 de-DE 0.415 -> 0.418 pt-PT 0.423 -> 0.424 es-ES 0.420 -> 0.420 nl-NL 0.396 -> 0.393 en-US 0.623 -> 0.623 English moved to multilingual: MASSIVE en-US 1 of 16,521 (all splits); CLINC150, BANKING77, AG News, SST-2 and a synthetic support-ticket set none. No utterance of any locale moves to English. --- laya/lang.py | 31 +++++++++++++++++++++---------- tests/test_router.py | 26 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/laya/lang.py b/laya/lang.py index 58ea55ea7..75f6cd82a 100644 --- a/laya/lang.py +++ b/laya/lang.py @@ -221,6 +221,10 @@ # ... van" is ordinary Dutch, and deduping it moved 14 of those rows while rescuing no English one. _EN_COLLISION_WORDS = {"come", "son", "do", "care", "todo", "im", "per", "plus"} & { w for lg, sw in _STOP.items() if lg != "en" for w in sw} +# Every word a non-English list holds. In text with no English function word, two different ones are +# evidence the text is not English, even when it is too thin to say which language (#54). One is not: +# English uses `do`, `todo` and `van` too ("how do my health benefits work"). +_OTHER_WORDS = frozenset(w for lg, words in _STOP.items() if lg != "en" for w in words) _WORD = re.compile(r"[^\W\d_]+", re.UNICODE) # A token whose dot or @ joins word characters is an identifier, not prose: `github.com`, @@ -433,11 +437,11 @@ def _english_rescued_by_words(words: List[str], diac_rate: float) -> bool: def latin_profile(text: str) -> Dict[str, object]: """Evidence behind the Latin-script language guess. - Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and - `looks_non_english`. The latter can be true for non-English diacritics or an overlapping - Swedish-Danish marker even when this heuristic cannot name the language. `analyse` needs the - evidence and not just the verdict, because "undecided" and "English" are different answers - and only one of them is safe to send to the English checkpoint. + Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate`, + `looks_non_english` and `other_evidence`. `looks_non_english` can be true for non-English + diacritics or an overlapping Swedish-Danish marker even when this heuristic cannot name the + language. `analyse` needs the evidence and not just the verdict, because "undecided" and + "English" are different answers and only one of them is safe to send to the English checkpoint. A non-English language is only named when it matched at least one word that no other list claims: shared function words alone (`la`, `e`, `o`) identify no particular language. @@ -451,10 +455,10 @@ def latin_profile(text: str) -> Dict[str, object]: nordic_overlap = bool(set(words) & _NORDIC_OVERLAP_WORDS) and not bool(set(words) & _EN_ONLY_WORDS) if 1 < len(words) < 4 and set(words) & _SHORT_SWEDISH_WORDS: return {"language": "sv", "english_hits": 0, "diacritic_rate": diac_rate, - "looks_non_english": non_english} + "looks_non_english": non_english, "other_evidence": False} if len(words) < 4: return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate, - "looks_non_english": non_english or nordic_overlap} + "looks_non_english": non_english or nordic_overlap, "other_evidence": False} # A collision word counts once however often it repeats; every other word counts its hits. counts = Counter(words) @@ -486,8 +490,11 @@ def latin_profile(text: str) -> Dict[str, object]: lang = best_lg elif en and (not non_english or _english_rescued_by_words(words, diac_rate)): lang = "en" + # a non-English letter too rare for the rate counts as evidence too + other = diac > 0 or len(_OTHER_WORDS.intersection(words)) >= 2 return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate, - "looks_non_english": non_english or (lang is None and nordic_overlap)} + "looks_non_english": non_english or (lang is None and nordic_overlap), + "other_evidence": other} def guess_latin_language(text: str) -> Optional[str]: @@ -737,9 +744,13 @@ def _analyse_text(text: str) -> Dict[str, object]: # Undecided is not English. Treating it as English sent every Latin-script language we hold no # stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the # language, non-English letters or a shared Swedish-Danish marker can still prefer the - # multilingual checkpoint; text with neither signal still goes to the English one. + # multilingual checkpoint. So is weaker evidence of another language at four or more words, + # where text left undecided has no English function word, since one would have named it `en` + # (#54). Text with none of these, like short English and English commands without a function + # word, still goes to the English one. undecided = lang is None - english = lang == "en" or (undecided and not prof_lat["looks_non_english"]) + english = lang == "en" or (undecided and not prof_lat["looks_non_english"] + and not prof_lat["other_evidence"]) return {"script": "latin", "script_profile": prof, "language": lang, "is_english": english, "language_undecided": undecided, "diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4), diff --git a/tests/test_router.py b/tests/test_router.py index 44d240092..b44a80cde 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -739,6 +739,32 @@ def _ms(state, reps=9): # German words that Spanish (`es`) or French (`du`) also claim would stop naming those languages check("latin_lang/spanish es stays evidence", guess_latin_language("que hora es en australia"), "es") check("latin_lang/french du stays evidence", guess_latin_language("baisse le volume du haut-parleur"), "fr") +# Undecided at four or more words (#54). Such text has no English function word, since one would name +# it `en`. Two different words from other languages' lists, or a non-English letter too rare for the +# rate, are evidence it is not English even when too thin to say which language. MASSIVE test utterances: +for text in ["streiche alle meine geplanten termine", # de, `alle` (it) and `meine` (de) + "me gustaría escuchar algunos buenos chistes divertidos"]: # es, one `í` + check("latin_lang/undecided with evidence " + text, guess_latin_language(text), None) + check("route/undecided with evidence " + text, _r_lat.route(text).model, "multilingual") +# One word is not enough, and neither is one word twice: English uses `do` and `las` too (both from the +# English texts checked on #286), so such text follows `default`, Dutch and Portuguese with it +for text in ["how do my health benefits work", "las vegas weather today", + "vertel me wat er gebeurt op instagram", # nl, `op` + "verificar qualquer email da amazon", # pt, `da` + "apaga la luz de la cocina"]: # es, `la` twice + check("route/one list word follows default " + text, _r_ml.route(text).model, "multilingual") + check("route/one list word stock default " + text, _r_lat.route(text).model, "english") +# Under four words the text is not scored, so two such words are no evidence yet +check("route/undecided three words follow default", _r_lat.route("wecke mich auf").model, "english") +# No evidence, no move: English with no function word stays English (the first three are from the 20,000 +# English texts checked on #286, the last from en-US MASSIVE) ... +for text in ["tell me my current savings account's interest rate", "play recently added music", + "how many days before my credit card arrives", "turn off lobby light"]: + check("route/english without a function word " + text, _r_lat.route(text).model, "english") +# ... and so, kept visible on purpose, does a language with no list here and no accents to show for it. +# `lang_guess=` or `Router(default="multilingual")` is the way round it. +check("route/KNOWN GAP indonesian without evidence", + _r_lat.route("tolong mainkan lagu dari bruno mars").model, "english") # ------------------------------------------------------------------ accented loanwords in English (#337) # The diacritic rate is measured over every character, so one `é` in a short English sentence From 6af2ad8b5b25cdffc43bf8d92263b1e76e05c2b8 Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Mon, 28 Sep 2026 11:17:11 +0530 Subject: [PATCH 2/9] fix(router): name the evidence when undecided text leaves the default Undecided Latin text that goes to multilingual on other-language evidence has a diacritic rate under NON_EN_DIACRITIC_RATE, so the reason read "language not identified but 0% non-English letters", which says the opposite of why it moved. It now names the evidence: two function words of other languages, or a non-English letter. --- laya/router.py | 9 ++++++++- tests/test_router.py | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/laya/router.py b/laya/router.py index 7853ec9a9..b0cb0f951 100644 --- a/laya/router.py +++ b/laya/router.py @@ -45,7 +45,7 @@ validate_timeout, ) from .hooks import _SKIP_DEFAULTS -from .lang import analyse +from .lang import NON_EN_DIACRITIC_RATE, analyse # The hub repo bundles all three checkpoints; only the requested subfolder is downloaded. BUNDLE_REPO = "convaiinnovations/laya" @@ -1276,6 +1276,13 @@ def _route( "the English checkpoint cannot read it" % (det["language"], det["mixed_segment"][:60])) elif det["language"]: reason = "Latin script but language looks like %r, not English" % det["language"] + elif float(det["diacritic_rate"]) < NON_EN_DIACRITIC_RATE: + # Too few non-English letters to count alone: `analyse` routed it on four or more + # words with no English function word and some evidence of another language (#54). + evidence = ("a non-English letter" if det["diacritic_rate"] + else "two function words of other languages") + reason = ("Latin script, language not identified; no English function word, but %s; " + "not safe for the English checkpoint" % evidence) else: # Unidentified Latin-script language: routed on the non-English letters alone, # because no stopword list here covers it. diff --git a/tests/test_router.py b/tests/test_router.py index b44a80cde..12beb61c4 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -746,6 +746,10 @@ def _ms(state, reps=9): "me gustaría escuchar algunos buenos chistes divertidos"]: # es, one `í` check("latin_lang/undecided with evidence " + text, guess_latin_language(text), None) check("route/undecided with evidence " + text, _r_lat.route(text).model, "multilingual") +check("route/undecided evidence reason names the words", "two function words of other languages" in + _r_lat.route("streiche alle meine geplanten termine").reason, True) +check("route/undecided evidence reason names the letter", "a non-English letter" in + _r_lat.route("me gustaría escuchar algunos buenos chistes divertidos").reason, True) # One word is not enough, and neither is one word twice: English uses `do` and `las` too (both from the # English texts checked on #286), so such text follows `default`, Dutch and Portuguese with it for text in ["how do my health benefits work", "las vegas weather today", From 8d0188a111d9774e3584c0ae268eea8176866c28 Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Mon, 28 Sep 2026 11:17:11 +0530 Subject: [PATCH 3/9] docs(readme): say when undecided text leaves the default The `default` paragraph now says that from four words up, text with no English function word but two different words from other languages' lists, or a non-English letter, goes to the multilingual checkpoint, and why one such word is not enough. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 787206243..1db577c29 100644 --- a/README.md +++ b/README.md @@ -578,7 +578,7 @@ router.route({"body": "Der Kunde wurde zweimal belastet"}, questions).reason # "Latin script but language looks like 'de', not English" ``` -Very short Latin-script text often carries nothing that identifies its language (`"Quero cancelar"`, `"Esqueci minha senha"`). Such text goes to `default`, which is `"multilingual"` since 0.4.0. Text the detector *can* place is unaffected: identified English still goes to the English checkpoint. +Very short Latin-script text often carries nothing that identifies its language (`"Quero cancelar"`, `"Esqueci minha senha"`). Such text goes to `default`, which is `"multilingual"` since 0.4.0. Text the detector *can* place is unaffected: identified English still goes to the English checkpoint. From four words up, text that names no language and has no English function word, but does contain two different words from other languages' lists or a non-English letter, goes to the multilingual checkpoint regardless of `default` (`"streiche alle meine geplanten termine"`). One such word is not enough, because English uses `do`, `todo` and `van` too. ```python router = Router() # default="multilingual" From c2949f58d6faa815626a4d3c4a4c7bb0661bf779 Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Sat, 3 Oct 2026 15:46:29 +0530 Subject: [PATCH 4/9] feat(lang): count one function word English never uses as evidence of another language Undecided Latin text of four or more words left the default only with two distinct list words or a non-English letter. Indonesian, Malay, Javanese, Swahili, Tagalog, Afrikaans and Welsh had no list, so their plain ASCII carried no evidence at all and went to the English checkpoint. - Add function-word lists for those seven languages. A word another list already holds is left out, so no other language's score changes, and so is a word in more than 1 in 2,000 English train documents. - One list word is now evidence on its own when it occurs in none of 212,592 English documents (train splits of CLINC150, BANKING77, SST-2 and AG News). Those 568 words are `_FOREIGN_WORDS` in laya/lang.py, generated by scripts/build_foreign_words.py (pinned revisions, --check), which also writes laya-ts/src/foreign-words.ts. lang.py stays self-contained, so research/evals/massive_route_comparison.py can still load it from any commit. A word English uses too (`do`, `van`, `per`) still needs a second one. - The route reason says "function words of other languages", since one can now be enough. MASSIVE en-US test: 0 of 2,974 rows move (main: 0). Across 61,830 English texts one moves, the same one as before (`play do re mi`). Co-Authored-By: Claude Opus 5.5 --- laya-ts/src/foreign-words.ts | 44 +++++++++++++ laya/lang.py | 87 +++++++++++++++++++++++++- laya/router.py | 2 +- scripts/build_foreign_words.py | 109 +++++++++++++++++++++++++++++++++ tests/test_router.py | 36 ++++++++--- 5 files changed, 265 insertions(+), 13 deletions(-) create mode 100644 laya-ts/src/foreign-words.ts create mode 100644 scripts/build_foreign_words.py diff --git a/laya-ts/src/foreign-words.ts b/laya-ts/src/foreign-words.ts new file mode 100644 index 000000000..30e10d6dd --- /dev/null +++ b/laya-ts/src/foreign-words.ts @@ -0,0 +1,44 @@ +/** Function words of other languages that English text does not use. + * + * Generated by `scripts/build_foreign_words.py`; do not edit by hand. Same words as `_FOREIGN_WORDS` + * in `laya/lang.py`: the words of the non-English lists found in none of 212,592 English documents. + * 568 words. + */ +export const FOREIGN_WORDS: ReadonlySet = new Set(` +abar abbiamo aceasta această acest acesta acum adakah adesso agora ainda ajke akan akhon ako aku +algo alguem alguém alla alle alors amader amake amma amra ancaq anche ancora apakah apnake apnar +apnara apni appen aqui aquí arep artiq artıq asbe asseblief atau aterbetala aterbetalning até +auch auf aunque avete awak bado bagaimana bakal behover behöver belum berapa besok bhai bhalo +bien bila bir bisa bitte ble blivit boleh bolte bolun bukan bukas bydd cadê chaina ciao como +consigo cuando daardie dagli daha dalam dalla dallo dapat dari dat ddim debiterade degli dello +dengan depois dessa detta deu deux deve devo devono deyil deze dheweke dhonnobad dich diese +diesen dieser dieses dilam dimana dir dite diye diyechi dois doit donc donde dorkar duibar după +durung də efo eger ei eich ein eine einen einer eita ekbar ekhon ekhono ekta elles endi entao +então esok esta estamos estava esto estou está faktura fel ferot ficou fiz foarte foi fois fost +fra från fue fueron får für fără gaan gan ganger geche gelem gese gibt già gli gostaria gyda +gång gånger gör göra görə habe haben harus heddiw hefyd hemos hendak heute hierdie hii hittar +hiyo hizi hjalp hjalpa hjälp hjälpa hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle huu +här həm hər ich ieri ikaw iki iku ile ilə ingin ini installningarna installningen +inställningarna inställningen inte isso isto isə iyan iyon jabe jetzt jika jodi jonno jours juga +julle jy já kabeh kailan kalau kalke kama kami kamu kanggo kann kannst kapan karena karo katika +keno kepiye kerana kesho keu kharap khub kibhabe kichu kila kintu kita kivabe kokhon kommer +korbo korchi kore koreche korechi korlam korsi korte korun kothay koto kowe kraschar kuwi +kvittot kwa kwamba lagbe lagi lakin lakini lini losenord lui lösenord maar mahu mais mana maneh +mau mayroon meine meinem meinen meiner menyang mereka mes meu meus mimi minha minhas moddhe mois +molto muito mung más måste môre mən naar nach nai naman namin nang nanging nani nao nar natin +nawr necesito negli nel nella neng nenhum nenhuma neu ngapi ngayon nhw nicht nie niet nila ning +ninguem ninguém nini niya niye noch nog noite nossa nosso nous nya nyinyi não när nästa nə +obrigada obrigado och också oder oggi oi olan oleh olmasa olub olá ond onde onek onlar ontem oor +oppnar ora paano pabo pada paini paket paki pakiusap parben parbo parchi parchina pela pelo +pentru perche peux peyechi pira piye più poco pode podem por porque pourquoi preciso pryd puede +pueden pwy până quand quando quero questa questo quiero roedd saan sahaja saja saka sampeyan +sapa sathe saya scusa sedang sekarang sempre semua sesuk shathe shob shomossa siapa sich sila +sisi siya siz skicka skulle sobre somossa sono sonra sparningen spårningen stato sua sudah sul +sulla sulle sunt sut são säger să sən tader tafadhali tahole tambem também tapi tarde tayo telah +tena tengo tenho tetapi theke tidak tiene tienen tolong tomar tomra tous toute trebuie très três +tudo tulung tumi tuo tva två tá ucun uga uit untuk upp uppdaterats utawa valo vandag var veut +veux vill vipi voces você vocês voglio voor vorrei vous vreau vă və waar wae wala wann wanneer +wao wapi welche werden wewe wir wird worden wordt wrth wurde yalniz yalnız ydy yeye yfory yn +yote yox yoxdur yw zijn zum zur är återbetala återbetalning çox è é être în öppnar üçün și ți +əgər +`.split(/\s+/).filter((w) => w.length > 0)); diff --git a/laya/lang.py b/laya/lang.py index 75f6cd82a..3663868af 100644 --- a/laya/lang.py +++ b/laya/lang.py @@ -165,6 +165,42 @@ "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox", "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma", "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"}, + + # Indonesian, Malay, Javanese, Swahili, Tagalog, Afrikaans and Welsh are mostly typed in plain ASCII, + # so with no list they left no evidence at all and went to the English checkpoint (#54). Two kinds of + # word are left out. A word another list already holds (`di`, `na`, `ni`, `si`, `ya`, `het`, `op`, + # `ar`) would change that language's score. A word found in more than 1 in 2,000 English documents + # (`dan`, `yen`, `au`, `sa`, `mo`, `pa`, `met`, `van`, `os`, `hi`, `mae`, `y`) would be English + # evidence as often as foreign. Indonesian and Malay share most of theirs, so a shared word names + # neither and only `mau`, `mahu`, `kamu` and the like tell them apart. + "id": {"yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "kamu", "anda", "dia", + "kami", "kita", "mereka", "apa", "apakah", "bagaimana", "kapan", "dimana", "mana", "siapa", + "berapa", "tidak", "bukan", "akan", "sudah", "belum", "sedang", "bisa", "dapat", "harus", + "mau", "ingin", "tolong", "juga", "atau", "tetapi", "tapi", "pada", "dalam", "oleh", + "karena", "jika", "kalau", "ada", "semua", "lagi", "saja", "nya", "besok", "sekarang"}, + "ms": {"yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "awak", "anda", "dia", + "kami", "kita", "mereka", "apa", "adakah", "bagaimana", "bila", "mana", "siapa", "berapa", + "tidak", "tak", "bukan", "akan", "sudah", "telah", "boleh", "mahu", "hendak", "tolong", + "juga", "atau", "tetapi", "pada", "dalam", "oleh", "kerana", "jika", "kalau", "ada", + "semua", "lagi", "sahaja", "esok", "sekarang"}, + "jv": {"aku", "kowe", "sampeyan", "dheweke", "iki", "kuwi", "iku", "lan", "karo", "ing", "menyang", + "saka", "kanggo", "apa", "piye", "kepiye", "kapan", "endi", "sapa", "pira", "ora", "wis", + "durung", "arep", "bakal", "isa", "iso", "bisa", "gelem", "tulung", "uga", "utawa", + "nanging", "ana", "kabeh", "maneh", "wae", "mung", "kang", "sing", "ning", "neng", "nang", + "sesuk", "saiki"}, + "sw": {"wa", "za", "kwa", "katika", "hii", "hiyo", "huu", "hizi", "mimi", "wewe", "yeye", "sisi", + "nyinyi", "wao", "nini", "vipi", "lini", "wapi", "nani", "ngapi", "sana", "tafadhali", + "kwamba", "lakini", "kama", "bado", "tena", "pia", "yote", "kila", "leo", "kesho"}, + "tl": {"ang", "ng", "mga", "ay", "ako", "ikaw", "ka", "siya", "kami", "tayo", "sila", "ko", "niya", + "namin", "natin", "nila", "ito", "iyan", "iyon", "ano", "paano", "kailan", "saan", "sino", + "ilan", "hindi", "oo", "po", "naman", "lang", "rin", "ngayon", "bukas", "kung", "mayroon", + "wala", "gusto", "paki", "pakiusap"}, + "af": {"en", "nie", "ek", "jy", "hy", "sy", "ons", "julle", "hulle", "vir", "uit", "oor", "wat", + "hoe", "wanneer", "waar", "hoeveel", "asseblief", "kan", "sal", "moet", "wil", "gaan", "dit", + "hierdie", "daardie", "nog", "vandag", "môre"}, + "cy": {"yr", "yn", "gyda", "wrth", "oes", "ydy", "yw", "roedd", "bydd", "fy", "dy", "eich", "beth", + "sut", "pryd", "ble", "pwy", "faint", "dim", "ddim", "ond", "neu", "hefyd", "nawr", "heddiw", + "yfory", "chi", "ti", "nhw", "fe", "gan", "efo"}, } # Short support fragments often consist of only two or three words, so they do not reach the @@ -222,9 +258,53 @@ _EN_COLLISION_WORDS = {"come", "son", "do", "care", "todo", "im", "per", "plus"} & { w for lg, sw in _STOP.items() if lg != "en" for w in sw} # Every word a non-English list holds. In text with no English function word, two different ones are -# evidence the text is not English, even when it is too thin to say which language (#54). One is not: -# English uses `do`, `todo` and `van` too ("how do my health benefits work"). +# evidence the text is not English, even when it is too thin to say which language (#54). One is not +# when English uses it too: `do`, `todo` and `van` ("how do my health benefits work"). One that +# English text never uses (`saya`, `kowe`, `nicht`) is, and `_FOREIGN_WORDS` holds those. _OTHER_WORDS = frozenset(w for lg, words in _STOP.items() if lg != "en" for w in words) +# The words of `_OTHER_WORDS` found in none of 212,592 English documents (the train splits of CLINC150, +# BANKING77, SST-2 and AG News). Generated by `scripts/build_foreign_words.py`, which rewrites this set +# and `laya-ts/src/foreign-words.ts`; `--check` says whether they are current. A word added to a list is +# not here until the script runs again, so it needs a second word like `do` does: the safe side. +_FOREIGN_WORDS = frozenset(""" + abar abbiamo aceasta această acest acesta acum adakah adesso agora ainda ajke akan akhon ako + aku algo alguem alguém alla alle alors amader amake amma amra ancaq anche ancora apakah + apnake apnar apnara apni appen aqui aquí arep artiq artıq asbe asseblief atau aterbetala + aterbetalning até auch auf aunque avete awak bado bagaimana bakal behover behöver belum + berapa besok bhai bhalo bien bila bir bisa bitte ble blivit boleh bolte bolun bukan bukas + bydd cadê chaina ciao como consigo cuando daardie dagli daha dalam dalla dallo dapat dari + dat ddim debiterade degli dello dengan depois dessa detta deu deux deve devo devono deyil + deze dheweke dhonnobad dich diese diesen dieser dieses dilam dimana dir dite diye diyechi + dois doit donc donde dorkar duibar după durung də efo eger ei eich ein eine einen einer eita + ekbar ekhon ekhono ekta elles endi entao então esok esta estamos estava esto estou está + faktura fel ferot ficou fiz foarte foi fois fost fra från fue fueron får für fără gaan gan + ganger geche gelem gese gibt già gli gostaria gyda gång gånger gör göra görə habe haben + harus heddiw hefyd hemos hendak heute hierdie hii hittar hiyo hizi hjalp hjalpa hjälp hjälpa + hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle huu här həm hər ich ieri ikaw iki + iku ile ilə ingin ini installningarna installningen inställningarna inställningen inte isso + isto isə iyan iyon jabe jetzt jika jodi jonno jours juga julle jy já kabeh kailan kalau + kalke kama kami kamu kanggo kann kannst kapan karena karo katika keno kepiye kerana kesho + keu kharap khub kibhabe kichu kila kintu kita kivabe kokhon kommer korbo korchi kore koreche + korechi korlam korsi korte korun kothay koto kowe kraschar kuwi kvittot kwa kwamba lagbe + lagi lakin lakini lini losenord lui lösenord maar mahu mais mana maneh mau mayroon meine + meinem meinen meiner menyang mereka mes meu meus mimi minha minhas moddhe mois molto muito + mung más måste môre mən naar nach nai naman namin nang nanging nani nao nar natin nawr + necesito negli nel nella neng nenhum nenhuma neu ngapi ngayon nhw nicht nie niet nila ning + ninguem ninguém nini niya niye noch nog noite nossa nosso nous nya nyinyi não när nästa nə + obrigada obrigado och också oder oggi oi olan oleh olmasa olub olá ond onde onek onlar ontem + oor oppnar ora paano pabo pada paini paket paki pakiusap parben parbo parchi parchina pela + pelo pentru perche peux peyechi pira piye più poco pode podem por porque pourquoi preciso + pryd puede pueden pwy până quand quando quero questa questo quiero roedd saan sahaja saja + saka sampeyan sapa sathe saya scusa sedang sekarang sempre semua sesuk shathe shob shomossa + siapa sich sila sisi siya siz skicka skulle sobre somossa sono sonra sparningen spårningen + stato sua sudah sul sulla sulle sunt sut são säger să sən tader tafadhali tahole tambem + também tapi tarde tayo telah tena tengo tenho tetapi theke tidak tiene tienen tolong tomar + tomra tous toute trebuie très três tudo tulung tumi tuo tva två tá ucun uga uit untuk upp + uppdaterats utawa valo vandag var veut veux vill vipi voces você vocês voglio voor vorrei + vous vreau vă və waar wae wala wann wanneer wao wapi welche werden wewe wir wird worden + wordt wrth wurde yalniz yalnız ydy yeye yfory yn yote yox yoxdur yw zijn zum zur är + återbetala återbetalning çox è é être în öppnar üçün și ți əgər +""".split()) _WORD = re.compile(r"[^\W\d_]+", re.UNICODE) # A token whose dot or @ joins word characters is an identifier, not prose: `github.com`, @@ -491,7 +571,8 @@ def latin_profile(text: str) -> Dict[str, object]: elif en and (not non_english or _english_rescued_by_words(words, diac_rate)): lang = "en" # a non-English letter too rare for the rate counts as evidence too - other = diac > 0 or len(_OTHER_WORDS.intersection(words)) >= 2 + other = (diac > 0 or len(_OTHER_WORDS.intersection(words)) >= 2 + or not _FOREIGN_WORDS.isdisjoint(words)) return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate, "looks_non_english": non_english or (lang is None and nordic_overlap), "other_evidence": other} diff --git a/laya/router.py b/laya/router.py index b0cb0f951..a45395f9f 100644 --- a/laya/router.py +++ b/laya/router.py @@ -1280,7 +1280,7 @@ def _route( # Too few non-English letters to count alone: `analyse` routed it on four or more # words with no English function word and some evidence of another language (#54). evidence = ("a non-English letter" if det["diacritic_rate"] - else "two function words of other languages") + else "function words of other languages") reason = ("Latin script, language not identified; no English function word, but %s; " "not safe for the English checkpoint" % evidence) else: diff --git a/scripts/build_foreign_words.py b/scripts/build_foreign_words.py new file mode 100644 index 000000000..85696c103 --- /dev/null +++ b/scripts/build_foreign_words.py @@ -0,0 +1,109 @@ +"""Build the list words that are evidence of another language on their own. + +Rewrites ``_FOREIGN_WORDS`` in ``laya/lang.py`` and ``laya-ts/src/foreign-words.ts``: every word +of the non-English stopword lists in ``laya/lang.py`` that occurs in none of the English documents +below. Those are the train splits of four public English corpora, at pinned revisions; their +test splits are where the rule's English cost is measured, so they are not read here. + +A word is read the way ``laya.lang`` reads one: a run of letters, lowercased. A list word that +English text does use (``do``, ``van``, ``per``) still needs a second, different one beside it. + +Usage:: + + pip install datasets + python scripts/build_foreign_words.py # download (cached by `datasets`), write + python scripts/build_foreign_words.py --check # exit 1 if the files differ + +Run it after changing a list. Until then a new word is not in the set, so it needs a second +word like any other: the safe side for English. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +import textwrap + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PY_OUT = os.path.join(REPO, "laya", "lang.py") +# The set's body in laya/lang.py; everything around it is hand-written and left alone. +PY_BLOCK = re.compile(r'(\n_FOREIGN_WORDS = frozenset\("""\n).*?(""".split\(\)\)\n)', re.S) +TS_OUT = os.path.join(REPO, "laya-ts", "src", "foreign-words.ts") + +# (dataset, config, revision, text column), train split of each. +SOURCES = [ + ("clinc/clinc_oos", "plus", "155b9c710419136e17307b80d0a13e68cd46b4ec", "text"), + ("mteb/banking77", None, "18072d2685ea682290f7b8924d94c62acc19c0b2", "text"), + ("stanfordnlp/sst2", None, "8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb", "sentence"), + ("fancyzhx/ag_news", None, "eb185aade064a813bc0b7f42de02595523103ca4", "text"), +] + + +def english_vocabulary(word_re): + from datasets import load_dataset + + seen, docs = set(), 0 + for name, config, revision, column in SOURCES: + for text in load_dataset(name, config, split="train", revision=revision)[column]: + seen.update(w.lower() for w in word_re.findall(text)) + docs += 1 + return seen, docs + + +def render_python(words, current): + body = textwrap.fill(" ".join(words), width=96, initial_indent=" ", subsequent_indent=" ") + out, n = PY_BLOCK.subn(lambda m: m.group(1) + body + "\n" + m.group(2), current) + if n != 1: + raise SystemExit("laya/lang.py: no `_FOREIGN_WORDS = frozenset(\"\"\"` block to rewrite") + return out + + +def render_ts(words, docs): + body = textwrap.fill(" ".join(words), width=96) + return """/** Function words of other languages that English text does not use. + * + * Generated by `scripts/build_foreign_words.py`; do not edit by hand. Same words as `_FOREIGN_WORDS` + * in `laya/lang.py`: the words of the non-English lists found in none of {docs:,} English documents. + * {n} words. + */ +export const FOREIGN_WORDS: ReadonlySet = new Set(` +{body} +`.split(/\\s+/).filter((w) => w.length > 0)); +""".format(docs=docs, n=len(words), body=body) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--check", action="store_true", help="compare instead of writing") + args = ap.parse_args(argv) + + sys.path.insert(0, REPO) + from laya.lang import _OTHER_WORDS, _WORD + + seen, docs = english_vocabulary(_WORD) + words = sorted(_OTHER_WORDS - seen) + stale = [] + for path in (PY_OUT, TS_OUT): + current = "" + if os.path.exists(path): + with open(path, encoding="utf-8") as f: + current = f.read() + text = render_python(words, current) if path == PY_OUT else render_ts(words, docs) + if current != text: + stale.append(os.path.relpath(path, REPO)) + if not args.check: + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(text) + print(f"{len(words)} of {len(_OTHER_WORDS)} list words occur in none of {docs:,} English documents") + if args.check and stale: + print("out of date: " + ", ".join(stale), file=sys.stderr) + return 1 + if not args.check: + print("rewrote: " + (", ".join(stale) or "nothing, already current")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_router.py b/tests/test_router.py index 12beb61c4..819fd8864 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -464,10 +464,13 @@ def _ms(state, reps=9): _r_ml = Router(default="multilingual") _r_en = Router(default="english") for text in ["Quero cancelar", "Esqueci minha senha", "Fui cobrado duas vezes", - "Produto veio quebrado, quero trocar", "refund me"]: + "refund me"]: check("route/undecided follows default " + text[:24], _r_ml.route(text).model, "multilingual") check("route/undecided stock default " + text[:24], _r_lat.route(text).model, "multilingual") check("route/undecided english override " + text[:24], _r_en.route(text).model, "english") +# From four words up, a function word English never uses is evidence of its own (#54): `quero` +check("route/undecided with a foreign word leaves the default", + _r_en.route("Produto veio quebrado, quero trocar").model, "multilingual") check("route/undecided reason names the default", "using default (multilingual)" in _r_ml.route("Esqueci minha senha").reason, True) # identified English is not undecided, so a non-English default leaves it alone @@ -746,14 +749,13 @@ def _ms(state, reps=9): "me gustaría escuchar algunos buenos chistes divertidos"]: # es, one `í` check("latin_lang/undecided with evidence " + text, guess_latin_language(text), None) check("route/undecided with evidence " + text, _r_lat.route(text).model, "multilingual") -check("route/undecided evidence reason names the words", "two function words of other languages" in +check("route/undecided evidence reason names the words", "function words of other languages" in _r_lat.route("streiche alle meine geplanten termine").reason, True) check("route/undecided evidence reason names the letter", "a non-English letter" in _r_lat.route("me gustaría escuchar algunos buenos chistes divertidos").reason, True) -# One word is not enough, and neither is one word twice: English uses `do` and `las` too (both from the -# English texts checked on #286), so such text follows `default`, Dutch and Portuguese with it +# One word that English uses too is not enough, and neither is one such word twice: `do` and `las` +# (both from the English texts checked on #286), so such text follows `default`, Portuguese with it for text in ["how do my health benefits work", "las vegas weather today", - "vertel me wat er gebeurt op instagram", # nl, `op` "verificar qualquer email da amazon", # pt, `da` "apaga la luz de la cocina"]: # es, `la` twice check("route/one list word follows default " + text, _r_ml.route(text).model, "multilingual") @@ -765,10 +767,26 @@ def _ms(state, reps=9): for text in ["tell me my current savings account's interest rate", "play recently added music", "how many days before my credit card arrives", "turn off lobby light"]: check("route/english without a function word " + text, _r_lat.route(text).model, "english") -# ... and so, kept visible on purpose, does a language with no list here and no accents to show for it. -# `lang_guess=` or `Router(default="multilingual")` is the way round it. -check("route/KNOWN GAP indonesian without evidence", - _r_lat.route("tolong mainkan lagu dari bruno mars").model, "english") +# ... while Indonesian, Malay, Javanese, Swahili, Tagalog, Afrikaans and Welsh, typed in plain ASCII, +# now have lists. One word English never uses is enough, and Dutch gains too (`uit` is Afrikaans as well). +# MASSIVE test utterances that went to the English checkpoint before: +for text in ["tolong mainkan lagu dari bruno mars", # id, `tolong` and `dari` + "batalkan alarm saya pukul tujuh pagi", # id, `saya` + "olly beritahu saya satu jenaka", # ms, `saya` + "olly ceritake aku guyonan", # jv, `aku` + "tafadhali washa plagi mahiri", # sw, `tafadhali` + "huwag mo akong gisingin bukas", # tl, `bukas` + "ken jy enige grappe", # af, `jy` + "paid a siarad heddiw", # cy, `heddiw` + "schakel de lampen uit", # nl, `uit` + "vertel me wat er gebeurt op instagram"]: # nl, `wat` and `op` + check("route/new list evidence " + text, _r_lat.route(text).model, "multilingual") +# A word of those lists that English uses too still needs a second one: en-US MASSIVE, `dim` and `kung` +for text in ["olly give me some dim light", "kung fu panda three"]: + check("route/english with a new list word " + text, _r_lat.route(text).model, "english") +# A word leaves _FOREIGN_WORDS with its list, or it would be evidence for no language at all +from laya.lang import _FOREIGN_WORDS, _OTHER_WORDS # noqa: E402 +check("latin_lang/foreign words all come from a list", sorted(_FOREIGN_WORDS - _OTHER_WORDS), []) # ------------------------------------------------------------------ accented loanwords in English (#337) # The diacritic rate is measured over every character, so one `é` in a short English sentence From 336452d6b4d84a214aed8c12101b7f62aed3bb48 Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Sat, 3 Oct 2026 15:46:29 +0530 Subject: [PATCH 5/9] feat(ts): port the undecided-evidence rule to laya-ts latinProfile() returns otherEvidence (a non-English letter, two distinct list words, or one word from FOREIGN_WORDS), analyseText() treats undecided text with it as not English, and the router reason names the evidence, as in laya/lang.py and laya/router.py. The seven new lists are copied word for word. A Danish sentence pinned as undecided is now named `af` (`kan`, `en`). It was not English before and is not now; only the name changed. Co-Authored-By: Claude Opus 5.5 --- laya-ts/src/lang.ts | 53 ++++++++++++++++++++++-- laya-ts/src/router.ts | 9 +++- laya-ts/tests/lang.test.ts | 3 +- laya-ts/tests/undecided-evidence.test.ts | 53 ++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 laya-ts/tests/undecided-evidence.test.ts diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts index a2c3e421e..8b99152d6 100644 --- a/laya-ts/src/lang.ts +++ b/laya-ts/src/lang.ts @@ -1,3 +1,5 @@ +import { FOREIGN_WORDS } from "./foreign-words.js"; + type ScriptRanges = Array<[string, Array<[number, number]>]>; const SCRIPT_RANGES: ScriptRanges = [ @@ -108,6 +110,35 @@ const STOP: Record> = { "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox", "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma", "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"]), + // Plain-ASCII languages that had no list (#54); laya/lang.py says what was left out and why. + id: new Set(["yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "kamu", "anda", + "dia", "kami", "kita", "mereka", "apa", "apakah", "bagaimana", "kapan", "dimana", "mana", "siapa", + "berapa", "tidak", "bukan", "akan", "sudah", "belum", "sedang", "bisa", "dapat", "harus", "mau", + "ingin", "tolong", "juga", "atau", "tetapi", "tapi", "pada", "dalam", "oleh", "karena", "jika", + "kalau", "ada", "semua", "lagi", "saja", "nya", "besok", "sekarang"]), + ms: new Set(["yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "awak", "anda", + "dia", "kami", "kita", "mereka", "apa", "adakah", "bagaimana", "bila", "mana", "siapa", "berapa", + "tidak", "tak", "bukan", "akan", "sudah", "telah", "boleh", "mahu", "hendak", "tolong", "juga", + "atau", "tetapi", "pada", "dalam", "oleh", "kerana", "jika", "kalau", "ada", "semua", "lagi", + "sahaja", "esok", "sekarang"]), + jv: new Set(["aku", "kowe", "sampeyan", "dheweke", "iki", "kuwi", "iku", "lan", "karo", "ing", + "menyang", "saka", "kanggo", "apa", "piye", "kepiye", "kapan", "endi", "sapa", "pira", "ora", + "wis", "durung", "arep", "bakal", "isa", "iso", "bisa", "gelem", "tulung", "uga", "utawa", + "nanging", "ana", "kabeh", "maneh", "wae", "mung", "kang", "sing", "ning", "neng", "nang", "sesuk", + "saiki"]), + sw: new Set(["wa", "za", "kwa", "katika", "hii", "hiyo", "huu", "hizi", "mimi", "wewe", "yeye", + "sisi", "nyinyi", "wao", "nini", "vipi", "lini", "wapi", "nani", "ngapi", "sana", "tafadhali", + "kwamba", "lakini", "kama", "bado", "tena", "pia", "yote", "kila", "leo", "kesho"]), + tl: new Set(["ang", "ng", "mga", "ay", "ako", "ikaw", "ka", "siya", "kami", "tayo", "sila", "ko", + "niya", "namin", "natin", "nila", "ito", "iyan", "iyon", "ano", "paano", "kailan", "saan", "sino", + "ilan", "hindi", "oo", "po", "naman", "lang", "rin", "ngayon", "bukas", "kung", "mayroon", "wala", + "gusto", "paki", "pakiusap"]), + af: new Set(["en", "nie", "ek", "jy", "hy", "sy", "ons", "julle", "hulle", "vir", "uit", "oor", + "wat", "hoe", "wanneer", "waar", "hoeveel", "asseblief", "kan", "sal", "moet", "wil", "gaan", + "dit", "hierdie", "daardie", "nog", "vandag", "môre"]), + cy: new Set(["yr", "yn", "gyda", "wrth", "oes", "ydy", "yw", "roedd", "bydd", "fy", "dy", "eich", + "beth", "sut", "pryd", "ble", "pwy", "faint", "dim", "ddim", "ond", "neu", "hefyd", "nawr", + "heddiw", "yfory", "chi", "ti", "nhw", "fe", "gan", "efo"]), }; const NON_EN_DIACRITICS = new Set( @@ -159,6 +190,9 @@ const EN_COLLISION_WORDS = new Set( ["come", "son", "do", "care", "todo", "im", "per", "plus"].filter((w) => Object.entries(STOP).some(([lg, sw]) => lg !== "en" && sw.has(w))), ); +// Every word a non-English list holds. In undecided text of four or more words, two different ones are +// evidence of another language, and so is one in FOREIGN_WORDS, which English text never uses (#54). +const OTHER_WORDS = new Set(Object.entries(STOP).filter(([lg]) => lg !== "en").flatMap(([, ws]) => [...ws])); // Short support fragments need a narrower vocabulary than the general four-word language guess. // Generic words such as "fel" and "hjälp" are deliberately omitted from this subset. @@ -314,6 +348,7 @@ export interface LatinProfile { englishHits: number; diacriticRate: number; looksNonEnglish: boolean; + otherEvidence: boolean; } export function latinProfile(text: string): LatinProfile { @@ -330,10 +365,13 @@ export function latinProfile(text: string): LatinProfile { const nordicOverlap = words.some((w) => NORDIC_OVERLAP_WORDS.has(w)) && !words.some((w) => EN_ONLY_WORDS.has(w)); if (words.length > 1 && words.length < 4 && words.some((w) => SHORT_SWEDISH_WORDS.has(w))) { - return { language: "sv", englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish }; + return { language: "sv", englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish, otherEvidence: false }; } if (words.length < 4) { - return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish || nordicOverlap }; + return { + language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish || nordicOverlap, + otherEvidence: false, + }; } // A collision word counts once however often it repeats; every other word counts its hits. const counts = new Map(); @@ -366,7 +404,13 @@ export function latinProfile(text: string): LatinProfile { } else if (en && (!nonEnglish || englishRescuedByWords(wordSet, diacRate))) { lang = "en"; } - return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish || (lang === null && nordicOverlap) }; + // a non-English letter too rare for the rate counts as evidence too + const otherWords = [...wordSet].filter((w) => OTHER_WORDS.has(w)); + const otherEvidence = diac > 0 || otherWords.length >= 2 || otherWords.some((w) => FOREIGN_WORDS.has(w)); + return { + language: lang, englishHits: en, diacriticRate: diacRate, + looksNonEnglish: nonEnglish || (lang === null && nordicOverlap), otherEvidence, + }; } export function guessLatinLanguage(text: string): string | null { @@ -490,7 +534,8 @@ function analyseText(text: string): AnalyseResult { const profLat = latinProfile(text); const lang = profLat.language; const undecided = lang === null; - const english = lang === "en" || (undecided && !profLat.looksNonEnglish); + // Undecided text with evidence of another language is not English either (#54), see laya/lang.py. + const english = lang === "en" || (undecided && !profLat.looksNonEnglish && !profLat.otherEvidence); return { script: "latin", scriptProfile: prof, language: lang, isEnglish: english, languageUndecided: undecided, diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts index a7d5114d7..deb0a0d23 100644 --- a/laya-ts/src/router.ts +++ b/laya-ts/src/router.ts @@ -1,4 +1,4 @@ -import { analyse, type AnalyseResult } from "./lang.js"; +import { NON_EN_DIACRITIC_RATE, analyse, type AnalyseResult } from "./lang.js"; import type { PredictOptions, QuestionDef, SystemOneResult } from "./agent.js"; import { checkMinConfidence, flagLowConfidence } from "./common.js"; import { decide, type DecideOptions, type DecisionResult } from "./structured.js"; @@ -557,6 +557,13 @@ export class Router extends HookRegistry { `(${JSON.stringify(det.mixedSegment.slice(0, 60))}); the English checkpoint cannot read it`; } else if (det.language) { reason = `Latin script but language looks like ${JSON.stringify(det.language)}, not English`; + } else if (det.diacriticRate < NON_EN_DIACRITIC_RATE) { + // Too few non-English letters to count alone: `analyse` routed it on four or more words with no + // English function word and some evidence of another language (#54). + const evidence = det.diacriticRate ? "a non-English letter" : "function words of other languages"; + reason = + `Latin script, language not identified; no English function word, but ${evidence}; ` + + "not safe for the English checkpoint"; } else { reason = `Latin script, language not identified but ${Math.round(100 * det.diacriticRate)}% ` + diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts index abfd66ce8..ba5a496dd 100644 --- a/laya-ts/tests/lang.test.ts +++ b/laya-ts/tests/lang.test.ts @@ -206,7 +206,8 @@ describe("lang", () => { it.each<[string, string | null]>([ ["Grüße aus Köln, wir melden uns wegen der Rechnung", "de"], ["sluk lyset i soveværelset", null], - ["kan jeg få en refundering for det dobbelte beløb", null], + // Danish has no list; `kan` and `en` are Afrikaans too, which names the wrong language but not English + ["kan jeg få en refundering for det dobbelte beløb", "af"], ["stäng av ljuset i sovrummet", null], ["jag vill ha en återbetalning för den dubbla avgiften", "sv"], ["The naïve façade needs a fresh coat of paint", null], diff --git a/laya-ts/tests/undecided-evidence.test.ts b/laya-ts/tests/undecided-evidence.test.ts new file mode 100644 index 000000000..e652ce9d3 --- /dev/null +++ b/laya-ts/tests/undecided-evidence.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { Router, guessLatinLanguage } from "../src/index.js"; +import { FOREIGN_WORDS } from "../src/foreign-words.js"; + +// Same cases as tests/test_router.py (#54): from four words up, undecided Latin text leaves the default +// only with evidence of another language. MASSIVE test utterances unless noted. +describe("undecided Latin text with evidence of another language", () => { + const router = new Router(); + + it.each([ + "streiche alle meine geplanten termine", // de, `alle` and `meine` + "me gustaría escuchar algunos buenos chistes divertidos", // es, one `í` + "tolong mainkan lagu dari bruno mars", // id, `tolong` and `dari` + "batalkan alarm saya pukul tujuh pagi", // id, `saya` + "olly beritahu saya satu jenaka", // ms, `saya` + "olly ceritake aku guyonan", // jv, `aku` + "tafadhali washa plagi mahiri", // sw, `tafadhali` + "huwag mo akong gisingin bukas", // tl, `bukas` + "ken jy enige grappe", // af, `jy` + "paid a siarad heddiw", // cy, `heddiw` + "schakel de lampen uit", // nl, `uit` + "vertel me wat er gebeurt op instagram", // nl, `wat` and `op` + ])("routes to multilingual: %s", (text) => { + expect(guessLatinLanguage(text)).toBeNull(); + expect(router.route(text, {}).model).toBe("multilingual"); + }); + + it("names the evidence in the reason", () => { + expect(router.route("streiche alle meine geplanten termine", {}).reason) + .toContain("function words of other languages"); + expect(router.route("me gustaría escuchar algunos buenos chistes divertidos", {}).reason) + .toContain("a non-English letter"); + }); + + it.each([ + "how do my health benefits work", // `do`, English texts checked on #286 + "las vegas weather today", + "verificar qualquer email da amazon", // pt, `da` alone + "olly give me some dim light", // `dim` (cy), en-US + "kung fu panda three", // `kung` (tl), en-US + "tell me my current savings account's interest rate", + "turn off lobby light", + "wecke mich auf", // under four words + ])("follows the default without it: %s", (text) => { + expect(router.route(text, {}).model).toBe("english"); + expect(new Router({ default: "multilingual" }).route(text, {}).model).toBe("multilingual"); + }); + + it("keeps the generated words", () => { + expect(FOREIGN_WORDS.has("saya")).toBe(true); + expect(FOREIGN_WORDS.has("do")).toBe(false); + }); +}); From c08d0534f8987b4fcd8a19e408479eb94bc93937 Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Sat, 3 Oct 2026 15:46:30 +0530 Subject: [PATCH 6/9] docs(readme): say that one word English never uses is evidence too Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1db577c29..0775b13f5 100644 --- a/README.md +++ b/README.md @@ -578,7 +578,7 @@ router.route({"body": "Der Kunde wurde zweimal belastet"}, questions).reason # "Latin script but language looks like 'de', not English" ``` -Very short Latin-script text often carries nothing that identifies its language (`"Quero cancelar"`, `"Esqueci minha senha"`). Such text goes to `default`, which is `"multilingual"` since 0.4.0. Text the detector *can* place is unaffected: identified English still goes to the English checkpoint. From four words up, text that names no language and has no English function word, but does contain two different words from other languages' lists or a non-English letter, goes to the multilingual checkpoint regardless of `default` (`"streiche alle meine geplanten termine"`). One such word is not enough, because English uses `do`, `todo` and `van` too. +Very short Latin-script text often carries nothing that identifies its language (`"Quero cancelar"`, `"Esqueci minha senha"`). Such text goes to `default`, which is `"multilingual"` since 0.4.0. Text the detector *can* place is unaffected: identified English still goes to the English checkpoint. From four words up, text that names no language and has no English function word, but does contain evidence of another language, goes to the multilingual checkpoint regardless of `default`: a non-English letter, two different words from other languages' lists (`"streiche alle meine geplanten termine"`), or one list word that English text never uses (`"batalkan alarm saya pukul tujuh pagi"`). Those words are `_FOREIGN_WORDS` in `laya/lang.py`, generated by `scripts/build_foreign_words.py`. One word that English uses too is not enough: `do`, `todo` and `van`. ```python router = Router() # default="multilingual" From 3a96047b39728c5346be6b9d315b95d4e1d13ddc Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Sat, 3 Oct 2026 16:15:09 +0530 Subject: [PATCH 7/9] feat(lang): lists for Danish, Norwegian, Finnish and Slovenian, and more plain-ASCII function words Most of what #600 still caught and this rule missed was plain-ASCII text whose function words no list held: Danish, Norwegian, Finnish and Slovenian had no list, and the Dutch, Romanian, Swahili, Malay, Indonesian, Javanese and Welsh lists lacked common pronouns, prepositions and question words (`mijn`, `cum`, `kuhusu`, `kepada`). - Add `da`, `nb`, `fi` and `sl` lists, and extend `nl`, `ro`, `id`, `ms`, `jv`, `sw` and `cy`. Same filters as before: no word another list holds, none in more than 1 in 2,000 English documents. `da` and `nb` also leave out every Swedish word, so Swedish keeps its name. Romanian keeps `pe` and `ca` out, as its list's comment asks. - Only function words were added, no MASSIVE-domain words such as "play" or "lamp". The words were chosen on MASSIVE validation, and test was read only for the final numbers. - Regenerated `_FOREIGN_WORDS`: 841 of 1,162 list words. MASSIVE test: en-US 0 of 2,974 rows move, no row moves to English, and checkpoint route accuracy goes from 86.55% to 88.54%. Across 61,830 English texts one moves (the same `play do re mi`). Co-Authored-By: Claude Opus 5.5 --- laya-ts/src/foreign-words.ts | 91 ++++++++++------- laya/lang.py | 184 +++++++++++++++++++++++------------ tests/test_router.py | 20 +++- 3 files changed, 193 insertions(+), 102 deletions(-) diff --git a/laya-ts/src/foreign-words.ts b/laya-ts/src/foreign-words.ts index 30e10d6dd..3be1639d6 100644 --- a/laya-ts/src/foreign-words.ts +++ b/laya-ts/src/foreign-words.ts @@ -2,43 +2,60 @@ * * Generated by `scripts/build_foreign_words.py`; do not edit by hand. Same words as `_FOREIGN_WORDS` * in `laya/lang.py`: the words of the non-English lists found in none of 212,592 English documents. - * 568 words. + * 841 words. */ export const FOREIGN_WORDS: ReadonlySet = new Set(` -abar abbiamo aceasta această acest acesta acum adakah adesso agora ainda ajke akan akhon ako aku -algo alguem alguém alla alle alors amader amake amma amra ancaq anche ancora apakah apnake apnar -apnara apni appen aqui aquí arep artiq artıq asbe asseblief atau aterbetala aterbetalning até -auch auf aunque avete awak bado bagaimana bakal behover behöver belum berapa besok bhai bhalo -bien bila bir bisa bitte ble blivit boleh bolte bolun bukan bukas bydd cadê chaina ciao como -consigo cuando daardie dagli daha dalam dalla dallo dapat dari dat ddim debiterade degli dello -dengan depois dessa detta deu deux deve devo devono deyil deze dheweke dhonnobad dich diese -diesen dieser dieses dilam dimana dir dite diye diyechi dois doit donc donde dorkar duibar după -durung də efo eger ei eich ein eine einen einer eita ekbar ekhon ekhono ekta elles endi entao -então esok esta estamos estava esto estou está faktura fel ferot ficou fiz foarte foi fois fost -fra från fue fueron får für fără gaan gan ganger geche gelem gese gibt già gli gostaria gyda -gång gånger gör göra görə habe haben harus heddiw hefyd hemos hendak heute hierdie hii hittar -hiyo hizi hjalp hjalpa hjälp hjälpa hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle huu -här həm hər ich ieri ikaw iki iku ile ilə ingin ini installningarna installningen -inställningarna inställningen inte isso isto isə iyan iyon jabe jetzt jika jodi jonno jours juga -julle jy já kabeh kailan kalau kalke kama kami kamu kanggo kann kannst kapan karena karo katika -keno kepiye kerana kesho keu kharap khub kibhabe kichu kila kintu kita kivabe kokhon kommer -korbo korchi kore koreche korechi korlam korsi korte korun kothay koto kowe kraschar kuwi -kvittot kwa kwamba lagbe lagi lakin lakini lini losenord lui lösenord maar mahu mais mana maneh -mau mayroon meine meinem meinen meiner menyang mereka mes meu meus mimi minha minhas moddhe mois -molto muito mung más måste môre mən naar nach nai naman namin nang nanging nani nao nar natin -nawr necesito negli nel nella neng nenhum nenhuma neu ngapi ngayon nhw nicht nie niet nila ning -ninguem ninguém nini niya niye noch nog noite nossa nosso nous nya nyinyi não när nästa nə -obrigada obrigado och också oder oggi oi olan oleh olmasa olub olá ond onde onek onlar ontem oor -oppnar ora paano pabo pada paini paket paki pakiusap parben parbo parchi parchina pela pelo -pentru perche peux peyechi pira piye più poco pode podem por porque pourquoi preciso pryd puede -pueden pwy până quand quando quero questa questo quiero roedd saan sahaja saja saka sampeyan -sapa sathe saya scusa sedang sekarang sempre semua sesuk shathe shob shomossa siapa sich sila -sisi siya siz skicka skulle sobre somossa sono sonra sparningen spårningen stato sua sudah sul -sulla sulle sunt sut são säger să sən tader tafadhali tahole tambem também tapi tarde tayo telah -tena tengo tenho tetapi theke tidak tiene tienen tolong tomar tomra tous toute trebuie très três -tudo tulung tumi tuo tva två tá ucun uga uit untuk upp uppdaterats utawa valo vandag var veut -veux vill vipi voces você vocês voglio voor vorrei vous vreau vă və waar wae wala wann wanneer -wao wapi welche werden wewe wir wird worden wordt wrth wurde yalniz yalnız ydy yeye yfory yn -yote yox yoxdur yw zijn zum zur är återbetala återbetalning çox è é être în öppnar üçün și ți -əgər +abar abbiamo aceasta această acest acesta acolo acum adakah adesso agora aici ainda ajke akan +akhon ako aku algo alguem alguém alla alle alors alsjeblieft alstublieft amader amake amma amra +ancaq anche ancora apakah apnake apnar apnara apni appen aqui aquí arep artiq artıq asbe +asseblief astazi atau aterbetala aterbetalning até auch auf aunque avete awak azi baada babagan +bado bagaimana bakal behover behöver belum berapa berapakah besok bhai bhalo bien bij bila +bilakah bir bisa bisakah bitte ble blivit boleh bolte bolun bukan bukas bydd cadê cand cati +chaina chdi ciao cine como consigo cuando daardie dadi dagli daha dalam dalla dallo dapat +dapatkah dari daripada dat ddim debiterade deg degli dello dengan denne depois dere deres despre +dessa detta dette deu deux deve devo devono deyil deze dhateng dheweke dhonnobad dich diese +diesen dieser dieses dilam dimana dimanakah diolch dir disse dite diye diyechi dois doit donc +donde dorkar duibar dupa după durung də efo eger ei eich ein eine einen einer eisiau eita ekbar +ekhon ekhono ekta elles endi entao então esok esta estamos estava esto estou está etta etter +että faktura fara fel ferot ficou fiz foarte foi fois fost fra från fue fueron fyddai får før +für fără gaan gallu gan ganger gani geche geen gelem gese gibt già gli gostaria graag gyda gyfer +gång gånger gör göra görə habe haben hadde haluaisin haluan hanen hanya hapa hapana harus havde +heb hebt heddiw heeft hefyd heidän hemos hendak hende hendes heute hierdie hii hij hingga hittar +hivi hiyo hizi hjalp hjalpa hjälp hjälpa hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle +huomenna huu hva hvad hvala hvem hvilke hvilken hvilket hvor hvordan hvorfor hvornaar hvornår +hwn hyn hän hänen här həm hər iawn ich ieri ik ikaw iki ikke iku ile ili ilə imorgen ingin ini +installningarna installningen inställningarna inställningen inte isio isso isto isə iti iyan +iyon jabe jaz jeg jer jeres jetzt jij jika jodi joku jonno jotain jours jouw juga julle jutri jy +já kabeh kabla kaikki kailan kaj kako kalau kalian kaliyan kalke kama kami kamu kanggo kann +kannst kanthi kapan karena karo katera kateri katero katika kdaj kdo kenapa kenen keno kepada +kepiye kerana kesho ketika keu kharap khub kibhabe kichu kiitos kila kintu kita kivabe kje +kokhon koliko kommer korbo korchi kore koreche korechi korlam korsi korte korun kothay koto kowe +kraschar kuhusu kuinka kuka kula kulo kuna kunne kutoka kuwi kvittot kwa kwamba kwenye lagbe +lagi lakin lakini lebih lini losenord lui luwih lösenord maar mahu mais mana manakah maneh masih +mau mayroon mboten meget meidän meine meinem meinen meiner mellem mellom menawa mengapa menika +menyang mereka mes meu meus mihin mij mijn miksi mikä milloin mimi mina minha minhas minua +minulle minun minä missa missä mista mistä mita miten mitä moddhe mois moj moja moje molto muito +mung mutta mye myos myös más måste môre mən naar nach nai nama naman namin nang nanging nani nao +naomba nar nataka natin nawr naš ndani ndiyo necesito negli nei nekaj nel nella neng nenhum +nenhuma neu ngapi ngayon nggih nhw nicht nie niet nila ninataka ning ninguem ninguém nini niya +niye nje njegov njen noch noen nog nogen noget noite nossa nosso nous nya nyinyi não nämä när +nästa nå når nə obrigada obrigado och också od oder oggi ogsaa også oi olan oleh oli olmasa olub +olá ond onde onek oni onko onlar ontem onze oor opo oppnar ora ovat paano pabo pada paini paket +paki pakiusap paling paljonko pamoja panjenengan parben parbo parchi parchina pela pelo pengin +pentru perche peux peyechi pira piye più plis poate pob poco pode podem por porque poti pourquoi +preciso prosim pryd puede pueden punika punya pwy până quand quando quero questa questo quiero +rhaid rhywbeth roedd saan saat sahaja saja saka saking samo sampai sampeyan sampun sangat sapa +sathe saya scusa sebelum sedaj sedang sekali sekarang semasa sempre semua semula sesudah sesuk +setelah shathe shob shomossa siapa siapakah sich siella siellä sila silakan sina sini sinua +sinulle sinun sinä sisi siya siz skal skicka skulle smo sobre soko somossa sono sonra sparningen +spårningen stato ste sua sudah sul sulla sulle sunt sut sydd são säger să sən taalla tader +tafadhali tahole takk tama taman tambem também tanaan tapi tarde tayo teidän telah tena tengo +tenho tentang tetapi theke tidak tiene tienen tolong tomar tomra tous toute trebuie très três +tudi tudo tulung tumi tuo tva tvoj tvoja två tá tämä tämän tänään täällä ucun uga ugi uit unde +unrhyw untuk upp uppdaterats utawa utawi valo vandag var vas vaš ved venligst vennligst veut +veux vill vipi voces você vocês voglio voisitko voitko voor vores vorrei vous vrea vreau vse vya +våre vă və waar waarom wae wakati wala wangu wann wanneer wao wapi wedi welche werden wewe wij +wir wird worden wordt wrth wurde yake yako yalniz yalnız yangu ydy yeye yfory yma yn yna yote +yox yoxdur yw zaidi zakaj zal zdaj zij zijn zum zur är återbetala återbetalning çox è é être în +öppnar üçün če și ți əgər `.split(/\s+/).filter((w) => w.length > 0)); diff --git a/laya/lang.py b/laya/lang.py index 3663868af..0518f6a18 100644 --- a/laya/lang.py +++ b/laya/lang.py @@ -116,8 +116,13 @@ # shared articles (`la fattura`) still name the language rather than stay undecided "nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei", "delle", "dello", "degli", "agli", "alle", "col"}, + # The pronouns and auxiliaries after `wordt` (`mijn`, `ik`, `heb`) carry most plain Dutch requests, + # which otherwise share only `de` and `het` with their neighbours. "nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan", - "door", "maar", "ook", "worden", "deze", "naar", "wordt"}, + "door", "maar", "ook", "worden", "deze", "naar", "wordt", + "mijn", "ik", "mij", "jij", "jouw", "uw", "zij", "wij", "hij", "hem", "haar", "onze", "hun", + "alsjeblieft", "alstublieft", "graag", "heb", "hebt", "heeft", "kun", "zal", "geen", "bij", + "tot", "welke", "waarom", "morgen", "er"}, # Swedish function words and common auxiliaries. Several overlap with English or German # (`i`, `kan`, `har`), so the distinctive words below are what lets Swedish text survive an # ASCII-normalising ticket pipeline without treating one stray Nordic letter as the only hint. @@ -133,10 +138,15 @@ "appen"}, # Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a # French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately - # left out for that reason, and the diacritic signal below carries the rest. + # left out for that reason, and the diacritic signal below carries the rest. Romanian typed without + # diacritics has no such signal, so the words after `nu` are ASCII spellings (`cand`, `fara`) and + # words no neighbour uses (`cum`, `cu`, `rog`). "ro": {"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără", "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această", - "acesta", "aceasta", "mi", "ți", "vă", "nu"}, + "acesta", "aceasta", "mi", "ți", "vă", "nu", + "despre", "cum", "rog", "cu", "mea", "mei", "azi", "astazi", "cand", "poti", "unde", "cine", + "cate", "cati", "imi", "iti", "vrea", "poate", "pot", "fara", "dupa", "pana", "maine", "aici", + "acolo"}, # Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali # keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English # and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence. Spelling @@ -166,31 +176,42 @@ "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma", "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"}, - # Indonesian, Malay, Javanese, Swahili, Tagalog, Afrikaans and Welsh are mostly typed in plain ASCII, - # so with no list they left no evidence at all and went to the English checkpoint (#54). Two kinds of - # word are left out. A word another list already holds (`di`, `na`, `ni`, `si`, `ya`, `het`, `op`, - # `ar`) would change that language's score. A word found in more than 1 in 2,000 English documents - # (`dan`, `yen`, `au`, `sa`, `mo`, `pa`, `met`, `van`, `os`, `hi`, `mae`, `y`) would be English - # evidence as often as foreign. Indonesian and Malay share most of theirs, so a shared word names - # neither and only `mau`, `mahu`, `kamu` and the like tell them apart. + # Indonesian, Malay, Javanese, Swahili, Tagalog, Afrikaans, Welsh, Danish, Norwegian, Finnish and + # Slovenian are often typed in plain ASCII, so with no list they left no evidence at all and went to + # the English checkpoint (#54). Two kinds of word are left out. A word another list already holds + # (`di`, `na`, `ni`, `si`, `ya`, `het`, `op`, `ar`, `du`, `je`) would change that language's score. A + # word found in more than 1 in 2,000 English documents (`dan`, `yen`, `au`, `sa`, `mo`, `pa`, `met`, + # `van`, `os`, `hi`, `mae`, `y`, `me`, `he`) would be English evidence as often as foreign. + # Indonesian and Malay share most of theirs, so a shared word names neither and only `mau`, `mahu`, + # `kamu` and the like tell them apart. "id": {"yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "kamu", "anda", "dia", "kami", "kita", "mereka", "apa", "apakah", "bagaimana", "kapan", "dimana", "mana", "siapa", "berapa", "tidak", "bukan", "akan", "sudah", "belum", "sedang", "bisa", "dapat", "harus", - "mau", "ingin", "tolong", "juga", "atau", "tetapi", "tapi", "pada", "dalam", "oleh", - "karena", "jika", "kalau", "ada", "semua", "lagi", "saja", "nya", "besok", "sekarang"}, + "mau", "ingin", "tolong", "juga", "atau", "tetapi", "tapi", "pada", "dalam", "oleh", "karena", + "jika", "kalau", "ada", "semua", "lagi", "saja", "nya", "besok", "sekarang", "tentang", + "kepada", "daripada", "mengapa", "kenapa", "silakan", "bisakah", "dapatkah", "sesudah", + "sebelum", "setelah", "hingga", "sampai", "sangat", "lebih", "paling", "masih", "hanya", + "punya", "kalian", "saat", "ketika", "sini"}, "ms": {"yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "awak", "anda", "dia", "kami", "kita", "mereka", "apa", "adakah", "bagaimana", "bila", "mana", "siapa", "berapa", - "tidak", "tak", "bukan", "akan", "sudah", "telah", "boleh", "mahu", "hendak", "tolong", - "juga", "atau", "tetapi", "pada", "dalam", "oleh", "kerana", "jika", "kalau", "ada", - "semua", "lagi", "sahaja", "esok", "sekarang"}, + "tidak", "tak", "bukan", "akan", "sudah", "telah", "boleh", "mahu", "hendak", "tolong", "juga", + "atau", "tetapi", "pada", "dalam", "oleh", "kerana", "jika", "kalau", "ada", "semua", "lagi", + "sahaja", "esok", "sekarang", "berapakah", "kepada", "daripada", "bilakah", "semula", "kenapa", + "mengapa", "siapakah", "dimanakah", "manakah", "sekali", "pun", "sangat", "lebih", "paling", + "masih", "hanya", "punya", "kalian", "semasa", "ketika", "sini"}, "jv": {"aku", "kowe", "sampeyan", "dheweke", "iki", "kuwi", "iku", "lan", "karo", "ing", "menyang", "saka", "kanggo", "apa", "piye", "kepiye", "kapan", "endi", "sapa", "pira", "ora", "wis", - "durung", "arep", "bakal", "isa", "iso", "bisa", "gelem", "tulung", "uga", "utawa", - "nanging", "ana", "kabeh", "maneh", "wae", "mung", "kang", "sing", "ning", "neng", "nang", - "sesuk", "saiki"}, + "durung", "arep", "bakal", "isa", "iso", "bisa", "gelem", "tulung", "uga", "utawa", "nanging", + "ana", "kabeh", "maneh", "wae", "mung", "kang", "sing", "ning", "neng", "nang", "sesuk", + "saiki", "opo", "ku", "kula", "soko", "babagan", "dadi", "kanthi", "menawa", "pengin", + "sampun", "mboten", "nggih", "kulo", "panjenengan", "niki", "menika", "punika", "saking", + "dhateng", "kaliyan", "ugi", "utawi", "luwih"}, "sw": {"wa", "za", "kwa", "katika", "hii", "hiyo", "huu", "hizi", "mimi", "wewe", "yeye", "sisi", "nyinyi", "wao", "nini", "vipi", "lini", "wapi", "nani", "ngapi", "sana", "tafadhali", - "kwamba", "lakini", "kama", "bado", "tena", "pia", "yote", "kila", "leo", "kesho"}, + "kwamba", "lakini", "kama", "bado", "tena", "pia", "yote", "kila", "leo", "kesho", "yangu", + "yako", "yake", "wangu", "wako", "kwenye", "cha", "vya", "gani", "kutoka", "kuhusu", "zaidi", + "hivi", "kuna", "nataka", "ninataka", "naomba", "sasa", "pamoja", "ndiyo", "hapana", "asante", + "ndani", "nje", "hapa", "pale", "wakati", "kabla", "baada", "ili"}, "tl": {"ang", "ng", "mga", "ay", "ako", "ikaw", "ka", "siya", "kami", "tayo", "sila", "ko", "niya", "namin", "natin", "nila", "ito", "iyan", "iyon", "ano", "paano", "kailan", "saan", "sino", "ilan", "hindi", "oo", "po", "naman", "lang", "rin", "ngayon", "bukas", "kung", "mayroon", @@ -200,7 +221,32 @@ "hierdie", "daardie", "nog", "vandag", "môre"}, "cy": {"yr", "yn", "gyda", "wrth", "oes", "ydy", "yw", "roedd", "bydd", "fy", "dy", "eich", "beth", "sut", "pryd", "ble", "pwy", "faint", "dim", "ddim", "ond", "neu", "hefyd", "nawr", "heddiw", - "yfory", "chi", "ti", "nhw", "fe", "gan", "efo"}, + "yfory", "chi", "ti", "nhw", "fe", "gan", "efo", "yma", "gyfer", "sydd", "dwi", "unrhyw", + "wedi", "gallu", "eisiau", "isio", "fyddai", "rhaid", "chdi", "fo", "hwn", "hon", "hyn", "yna", + "rhywbeth", "pob", "mwy", "iawn", "diolch", "plis"}, + # Danish and Norwegian (Bokmål) share most function words with each other, so a shared word names + # neither. A word they also share with Swedish (`det`, `med`, `på`, `av`, `som`, `har`) is left + # out as well as any other list's word, so Swedish text keeps its name. + "da": {"jeg", "hun", "ham", "hende", "jer", "dine", "hendes", "vores", "jeres", "deres", "og", "af", + "til", "ved", "hos", "før", "mellem", "hvad", "hvem", "hvor", "hvornår", "hvornaar", "hvordan", + "hvorfor", "hvilken", "hvilket", "hvilke", "er", "havde", "kunne", "skal", "ikke", "venligst", + "imorgen", "denne", "dette", "disse", "noget", "nogen", "også", "ogsaa", "meget", "mere", + "kun", "paa"}, + "nb": {"jeg", "hun", "dere", "meg", "deg", "ham", "dine", "våre", "deres", "og", "til", "ved", "hos", + "etter", "før", "mellom", "hva", "hvem", "hvor", "når", "hvordan", "hvorfor", "hvilken", + "hvilket", "hvilke", "er", "hadde", "kunne", "skal", "ikke", "nei", "takk", "vennligst", "nå", + "imorgen", "denne", "dette", "disse", "noe", "noen", "også", "mye", "kun"}, + "fi": {"minä", "mina", "sinä", "sina", "hän", "minun", "sinun", "hänen", "hanen", "meidän", "teidän", + "heidän", "minulle", "sinulle", "minua", "sinua", "mitä", "mita", "mikä", "mika", "missä", + "missa", "mistä", "mista", "mihin", "milloin", "miten", "kuinka", "kuka", "kenen", "miksi", + "paljonko", "onko", "voitko", "voisitko", "haluan", "haluaisin", "ole", "oli", "ovat", "tai", + "mutta", "että", "etta", "jos", "kun", "myös", "myos", "nyt", "tänään", "tanaan", "huomenna", + "kiitos", "kaikki", "jotain", "joku", "tämä", "tama", "tämän", "taman", "nämä", "nama", + "siellä", "siella", "täällä", "taalla"}, + "sl": {"jaz", "ona", "oni", "moj", "moja", "moje", "tvoj", "tvoja", "njegov", "njen", "naš", "vaš", + "vas", "kaj", "kdo", "kje", "kdaj", "kako", "zakaj", "koliko", "kateri", "katera", "katero", + "sem", "smo", "ste", "bo", "bom", "bi", "ker", "če", "z", "iz", "od", "pri", "prosim", "hvala", + "danes", "jutri", "zdaj", "sedaj", "tudi", "samo", "vse", "nekaj"}, } # Short support fragments often consist of only two or three words, so they do not reach the @@ -235,9 +281,9 @@ # way), though it still counts toward the total of a language that also matched a word of its own. _SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words} if sum(w in words for words in _STOP.values()) > 1} -# Danish is not in `_STOP`, but several of its common words also occur in the Swedish list. -# Do not let these words alone name a Danish sentence as Swedish; they remain useful score hits -# when another, more distinctive Swedish word is present. +# Several common Danish and Norwegian words are also in the Swedish list, and the `da` and `nb` lists +# leave them out. Do not let these words alone name a Danish sentence as Swedish; they remain useful +# score hits when another, more distinctive Swedish word is present. _NORDIC_OVERLAP_WORDS = { "hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi", } @@ -267,43 +313,61 @@ # and `laya-ts/src/foreign-words.ts`; `--check` says whether they are current. A word added to a list is # not here until the script runs again, so it needs a second word like `do` does: the safe side. _FOREIGN_WORDS = frozenset(""" - abar abbiamo aceasta această acest acesta acum adakah adesso agora ainda ajke akan akhon ako - aku algo alguem alguém alla alle alors amader amake amma amra ancaq anche ancora apakah - apnake apnar apnara apni appen aqui aquí arep artiq artıq asbe asseblief atau aterbetala - aterbetalning até auch auf aunque avete awak bado bagaimana bakal behover behöver belum - berapa besok bhai bhalo bien bila bir bisa bitte ble blivit boleh bolte bolun bukan bukas - bydd cadê chaina ciao como consigo cuando daardie dagli daha dalam dalla dallo dapat dari - dat ddim debiterade degli dello dengan depois dessa detta deu deux deve devo devono deyil - deze dheweke dhonnobad dich diese diesen dieser dieses dilam dimana dir dite diye diyechi - dois doit donc donde dorkar duibar după durung də efo eger ei eich ein eine einen einer eita - ekbar ekhon ekhono ekta elles endi entao então esok esta estamos estava esto estou está - faktura fel ferot ficou fiz foarte foi fois fost fra från fue fueron får für fără gaan gan - ganger geche gelem gese gibt già gli gostaria gyda gång gånger gör göra görə habe haben - harus heddiw hefyd hemos hendak heute hierdie hii hittar hiyo hizi hjalp hjalpa hjälp hjälpa - hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle huu här həm hər ich ieri ikaw iki - iku ile ilə ingin ini installningarna installningen inställningarna inställningen inte isso - isto isə iyan iyon jabe jetzt jika jodi jonno jours juga julle jy já kabeh kailan kalau - kalke kama kami kamu kanggo kann kannst kapan karena karo katika keno kepiye kerana kesho - keu kharap khub kibhabe kichu kila kintu kita kivabe kokhon kommer korbo korchi kore koreche - korechi korlam korsi korte korun kothay koto kowe kraschar kuwi kvittot kwa kwamba lagbe - lagi lakin lakini lini losenord lui lösenord maar mahu mais mana maneh mau mayroon meine - meinem meinen meiner menyang mereka mes meu meus mimi minha minhas moddhe mois molto muito - mung más måste môre mən naar nach nai naman namin nang nanging nani nao nar natin nawr - necesito negli nel nella neng nenhum nenhuma neu ngapi ngayon nhw nicht nie niet nila ning - ninguem ninguém nini niya niye noch nog noite nossa nosso nous nya nyinyi não när nästa nə - obrigada obrigado och också oder oggi oi olan oleh olmasa olub olá ond onde onek onlar ontem - oor oppnar ora paano pabo pada paini paket paki pakiusap parben parbo parchi parchina pela - pelo pentru perche peux peyechi pira piye più poco pode podem por porque pourquoi preciso - pryd puede pueden pwy până quand quando quero questa questo quiero roedd saan sahaja saja - saka sampeyan sapa sathe saya scusa sedang sekarang sempre semua sesuk shathe shob shomossa - siapa sich sila sisi siya siz skicka skulle sobre somossa sono sonra sparningen spårningen - stato sua sudah sul sulla sulle sunt sut são säger să sən tader tafadhali tahole tambem - também tapi tarde tayo telah tena tengo tenho tetapi theke tidak tiene tienen tolong tomar - tomra tous toute trebuie très três tudo tulung tumi tuo tva två tá ucun uga uit untuk upp - uppdaterats utawa valo vandag var veut veux vill vipi voces você vocês voglio voor vorrei - vous vreau vă və waar wae wala wann wanneer wao wapi welche werden wewe wir wird worden - wordt wrth wurde yalniz yalnız ydy yeye yfory yn yote yox yoxdur yw zijn zum zur är - återbetala återbetalning çox è é être în öppnar üçün și ți əgər + abar abbiamo aceasta această acest acesta acolo acum adakah adesso agora aici ainda ajke + akan akhon ako aku algo alguem alguém alla alle alors alsjeblieft alstublieft amader amake + amma amra ancaq anche ancora apakah apnake apnar apnara apni appen aqui aquí arep artiq + artıq asbe asseblief astazi atau aterbetala aterbetalning até auch auf aunque avete awak azi + baada babagan bado bagaimana bakal behover behöver belum berapa berapakah besok bhai bhalo + bien bij bila bilakah bir bisa bisakah bitte ble blivit boleh bolte bolun bukan bukas bydd + cadê cand cati chaina chdi ciao cine como consigo cuando daardie dadi dagli daha dalam dalla + dallo dapat dapatkah dari daripada dat ddim debiterade deg degli dello dengan denne depois + dere deres despre dessa detta dette deu deux deve devo devono deyil deze dhateng dheweke + dhonnobad dich diese diesen dieser dieses dilam dimana dimanakah diolch dir disse dite diye + diyechi dois doit donc donde dorkar duibar dupa după durung də efo eger ei eich ein eine + einen einer eisiau eita ekbar ekhon ekhono ekta elles endi entao então esok esta estamos + estava esto estou está etta etter että faktura fara fel ferot ficou fiz foarte foi fois fost + fra från fue fueron fyddai får før für fără gaan gallu gan ganger gani geche geen gelem gese + gibt già gli gostaria graag gyda gyfer gång gånger gör göra görə habe haben hadde haluaisin + haluan hanen hanya hapa hapana harus havde heb hebt heddiw heeft hefyd heidän hemos hendak + hende hendes heute hierdie hii hij hingga hittar hivi hiyo hizi hjalp hjalpa hjälp hjälpa + hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle huomenna huu hva hvad hvala hvem + hvilke hvilken hvilket hvor hvordan hvorfor hvornaar hvornår hwn hyn hän hänen här həm hər + iawn ich ieri ik ikaw iki ikke iku ile ili ilə imorgen ingin ini installningarna + installningen inställningarna inställningen inte isio isso isto isə iti iyan iyon jabe jaz + jeg jer jeres jetzt jij jika jodi joku jonno jotain jours jouw juga julle jutri jy já kabeh + kabla kaikki kailan kaj kako kalau kalian kaliyan kalke kama kami kamu kanggo kann kannst + kanthi kapan karena karo katera kateri katero katika kdaj kdo kenapa kenen keno kepada + kepiye kerana kesho ketika keu kharap khub kibhabe kichu kiitos kila kintu kita kivabe kje + kokhon koliko kommer korbo korchi kore koreche korechi korlam korsi korte korun kothay koto + kowe kraschar kuhusu kuinka kuka kula kulo kuna kunne kutoka kuwi kvittot kwa kwamba kwenye + lagbe lagi lakin lakini lebih lini losenord lui luwih lösenord maar mahu mais mana manakah + maneh masih mau mayroon mboten meget meidän meine meinem meinen meiner mellem mellom menawa + mengapa menika menyang mereka mes meu meus mihin mij mijn miksi mikä milloin mimi mina minha + minhas minua minulle minun minä missa missä mista mistä mita miten mitä moddhe mois moj moja + moje molto muito mung mutta mye myos myös más måste môre mən naar nach nai nama naman namin + nang nanging nani nao naomba nar nataka natin nawr naš ndani ndiyo necesito negli nei nekaj + nel nella neng nenhum nenhuma neu ngapi ngayon nggih nhw nicht nie niet nila ninataka ning + ninguem ninguém nini niya niye nje njegov njen noch noen nog nogen noget noite nossa nosso + nous nya nyinyi não nämä när nästa nå når nə obrigada obrigado och också od oder oggi ogsaa + også oi olan oleh oli olmasa olub olá ond onde onek oni onko onlar ontem onze oor opo oppnar + ora ovat paano pabo pada paini paket paki pakiusap paling paljonko pamoja panjenengan parben + parbo parchi parchina pela pelo pengin pentru perche peux peyechi pira piye più plis poate + pob poco pode podem por porque poti pourquoi preciso prosim pryd puede pueden punika punya + pwy până quand quando quero questa questo quiero rhaid rhywbeth roedd saan saat sahaja saja + saka saking samo sampai sampeyan sampun sangat sapa sathe saya scusa sebelum sedaj sedang + sekali sekarang semasa sempre semua semula sesudah sesuk setelah shathe shob shomossa siapa + siapakah sich siella siellä sila silakan sina sini sinua sinulle sinun sinä sisi siya siz + skal skicka skulle smo sobre soko somossa sono sonra sparningen spårningen stato ste sua + sudah sul sulla sulle sunt sut sydd são säger să sən taalla tader tafadhali tahole takk tama + taman tambem também tanaan tapi tarde tayo teidän telah tena tengo tenho tentang tetapi + theke tidak tiene tienen tolong tomar tomra tous toute trebuie très três tudi tudo tulung + tumi tuo tva tvoj tvoja två tá tämä tämän tänään täällä ucun uga ugi uit unde unrhyw untuk + upp uppdaterats utawa utawi valo vandag var vas vaš ved venligst vennligst veut veux vill + vipi voces você vocês voglio voisitko voitko voor vores vorrei vous vrea vreau vse vya våre + vă və waar waarom wae wakati wala wangu wann wanneer wao wapi wedi welche werden wewe wij + wir wird worden wordt wrth wurde yake yako yalniz yalnız yangu ydy yeye yfory yma yn yna + yote yox yoxdur yw zaidi zakaj zal zdaj zij zijn zum zur är återbetala återbetalning çox è é + être în öppnar üçün če și ți əgər """.split()) _WORD = re.compile(r"[^\W\d_]+", re.UNICODE) diff --git a/tests/test_router.py b/tests/test_router.py index 819fd8864..4bdceba07 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -767,9 +767,9 @@ def _ms(state, reps=9): for text in ["tell me my current savings account's interest rate", "play recently added music", "how many days before my credit card arrives", "turn off lobby light"]: check("route/english without a function word " + text, _r_lat.route(text).model, "english") -# ... while Indonesian, Malay, Javanese, Swahili, Tagalog, Afrikaans and Welsh, typed in plain ASCII, -# now have lists. One word English never uses is enough, and Dutch gains too (`uit` is Afrikaans as well). -# MASSIVE test utterances that went to the English checkpoint before: +# ... while languages often typed in plain ASCII now have lists, or more words in theirs (Dutch, +# Romanian). One word English never uses is enough. MASSIVE test utterances that went to the English +# checkpoint before: for text in ["tolong mainkan lagu dari bruno mars", # id, `tolong` and `dari` "batalkan alarm saya pukul tujuh pagi", # id, `saya` "olly beritahu saya satu jenaka", # ms, `saya` @@ -778,8 +778,15 @@ def _ms(state, reps=9): "huwag mo akong gisingin bukas", # tl, `bukas` "ken jy enige grappe", # af, `jy` "paid a siarad heddiw", # cy, `heddiw` - "schakel de lampen uit", # nl, `uit` - "vertel me wat er gebeurt op instagram"]: # nl, `wat` and `op` + "schakel de lampen uit", # nl, `uit` (Afrikaans too) + "vertel me wat er gebeurt op instagram", # nl, `wat` and `op` + "speel mijn rock afspeellijst", # nl, `mijn` + "cine a scris melodia asta", # ro, `cine` + "niambie kuhusu kengele zangu", # sw, `kuhusu` + "vil det regne denne uge", # da, `denne` + "slett alarmen jeg nettopp satte", # nb, `jeg` + "olly kerro minulle vitsi", # fi, `minulle` + "kateri alarmi so nastavljeni"]: # sl, `kateri` check("route/new list evidence " + text, _r_lat.route(text).model, "multilingual") # A word of those lists that English uses too still needs a second one: en-US MASSIVE, `dim` and `kung` for text in ["olly give me some dim light", "kung fu panda three"]: @@ -787,6 +794,9 @@ def _ms(state, reps=9): # A word leaves _FOREIGN_WORDS with its list, or it would be evidence for no language at all from laya.lang import _FOREIGN_WORDS, _OTHER_WORDS # noqa: E402 check("latin_lang/foreign words all come from a list", sorted(_FOREIGN_WORDS - _OTHER_WORDS), []) +# Danish and Norwegian leave out every word of the Swedish list, so Swedish keeps its name +check("latin_lang/da and nb share no word with sv", + sorted(w for lg in ("da", "nb") for w in _STOP[lg] & _STOP["sv"]), []) # ------------------------------------------------------------------ accented loanwords in English (#337) # The diacritic rate is measured over every character, so one `é` in a short English sentence From fbea2cc84b2017fa2e2a5822b4146f5a14b538f6 Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Sat, 3 Oct 2026 16:15:09 +0530 Subject: [PATCH 8/9] feat(ts): port the new and extended lists to laya-ts Same words, in the same order, as laya/lang.py. "vertel me wat er gebeurt op instagram" is now named `nl` rather than undecided, so the undecided pin moves to the two cases that stay undecided. Co-Authored-By: Claude Opus 5.5 --- laya-ts/src/lang.ts | 64 +++++++++++++++++++----- laya-ts/tests/undecided-evidence.test.ts | 17 ++++++- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts index 8b99152d6..8843edd80 100644 --- a/laya-ts/src/lang.ts +++ b/laya-ts/src/lang.ts @@ -78,7 +78,10 @@ const STOP: Record> = { "nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei", "delle", "dello", "degli", "agli", "alle", "col"]), nl: new Set(["het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan", - "door", "maar", "ook", "worden", "deze", "naar", "wordt"]), + "door", "maar", "ook", "worden", "deze", "naar", "wordt", + "mijn", "ik", "mij", "jij", "jouw", "uw", "zij", "wij", "hij", "hem", "haar", "onze", + "hun", "alsjeblieft", "alstublieft", "graag", "heb", "hebt", "heeft", "kun", "zal", "geen", "bij", + "tot", "welke", "waarom", "morgen", "er"]), // Keep Swedish routing in parity with Python. Distinctive words help ASCII-normalised ticket text; // shared Nordic words are tracked separately so Danish and Norwegian are not named Swedish. sv: new Set(["jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska", @@ -91,7 +94,10 @@ const STOP: Record> = { "blivit", "debiterade", "appen"]), ro: new Set(["și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără", "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această", - "acesta", "aceasta", "mi", "ți", "vă", "nu"]), + "acesta", "aceasta", "mi", "ți", "vă", "nu", + "despre", "cum", "rog", "cu", "mea", "mei", "azi", "astazi", "cand", "poti", "unde", + "cine", "cate", "cati", "imi", "iti", "vrea", "poate", "pot", "fara", "dupa", "pana", "maine", "aici", + "acolo"]), // Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali // keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English // and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence. @@ -115,20 +121,30 @@ const STOP: Record> = { "dia", "kami", "kita", "mereka", "apa", "apakah", "bagaimana", "kapan", "dimana", "mana", "siapa", "berapa", "tidak", "bukan", "akan", "sudah", "belum", "sedang", "bisa", "dapat", "harus", "mau", "ingin", "tolong", "juga", "atau", "tetapi", "tapi", "pada", "dalam", "oleh", "karena", "jika", - "kalau", "ada", "semua", "lagi", "saja", "nya", "besok", "sekarang"]), + "kalau", "ada", "semua", "lagi", "saja", "nya", "besok", "sekarang", "tentang", "kepada", "daripada", + "mengapa", "kenapa", "silakan", "bisakah", "dapatkah", "sesudah", "sebelum", "setelah", "hingga", + "sampai", "sangat", "lebih", "paling", "masih", "hanya", "punya", "kalian", "saat", "ketika", + "sini"]), ms: new Set(["yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "awak", "anda", "dia", "kami", "kita", "mereka", "apa", "adakah", "bagaimana", "bila", "mana", "siapa", "berapa", "tidak", "tak", "bukan", "akan", "sudah", "telah", "boleh", "mahu", "hendak", "tolong", "juga", "atau", "tetapi", "pada", "dalam", "oleh", "kerana", "jika", "kalau", "ada", "semua", "lagi", - "sahaja", "esok", "sekarang"]), + "sahaja", "esok", "sekarang", "berapakah", "kepada", "daripada", "bilakah", "semula", "kenapa", + "mengapa", "siapakah", "dimanakah", "manakah", "sekali", "pun", "sangat", "lebih", "paling", "masih", + "hanya", "punya", "kalian", "semasa", "ketika", "sini"]), jv: new Set(["aku", "kowe", "sampeyan", "dheweke", "iki", "kuwi", "iku", "lan", "karo", "ing", - "menyang", "saka", "kanggo", "apa", "piye", "kepiye", "kapan", "endi", "sapa", "pira", "ora", - "wis", "durung", "arep", "bakal", "isa", "iso", "bisa", "gelem", "tulung", "uga", "utawa", - "nanging", "ana", "kabeh", "maneh", "wae", "mung", "kang", "sing", "ning", "neng", "nang", "sesuk", - "saiki"]), - sw: new Set(["wa", "za", "kwa", "katika", "hii", "hiyo", "huu", "hizi", "mimi", "wewe", "yeye", - "sisi", "nyinyi", "wao", "nini", "vipi", "lini", "wapi", "nani", "ngapi", "sana", "tafadhali", - "kwamba", "lakini", "kama", "bado", "tena", "pia", "yote", "kila", "leo", "kesho"]), + "menyang", "saka", "kanggo", "apa", "piye", "kepiye", "kapan", "endi", "sapa", "pira", "ora", "wis", + "durung", "arep", "bakal", "isa", "iso", "bisa", "gelem", "tulung", "uga", "utawa", "nanging", "ana", + "kabeh", "maneh", "wae", "mung", "kang", "sing", "ning", "neng", "nang", "sesuk", "saiki", "opo", + "ku", "kula", "soko", "babagan", "dadi", "kanthi", "menawa", "pengin", "sampun", "mboten", "nggih", + "kulo", "panjenengan", "niki", "menika", "punika", "saking", "dhateng", "kaliyan", "ugi", "utawi", + "luwih"]), + sw: new Set(["wa", "za", "kwa", "katika", "hii", "hiyo", "huu", "hizi", "mimi", "wewe", "yeye", "sisi", + "nyinyi", "wao", "nini", "vipi", "lini", "wapi", "nani", "ngapi", "sana", "tafadhali", "kwamba", + "lakini", "kama", "bado", "tena", "pia", "yote", "kila", "leo", "kesho", "yangu", "yako", "yake", + "wangu", "wako", "kwenye", "cha", "vya", "gani", "kutoka", "kuhusu", "zaidi", "hivi", "kuna", + "nataka", "ninataka", "naomba", "sasa", "pamoja", "ndiyo", "hapana", "asante", "ndani", "nje", "hapa", + "pale", "wakati", "kabla", "baada", "ili"]), tl: new Set(["ang", "ng", "mga", "ay", "ako", "ikaw", "ka", "siya", "kami", "tayo", "sila", "ko", "niya", "namin", "natin", "nila", "ito", "iyan", "iyon", "ano", "paano", "kailan", "saan", "sino", "ilan", "hindi", "oo", "po", "naman", "lang", "rin", "ngayon", "bukas", "kung", "mayroon", "wala", @@ -137,8 +153,30 @@ const STOP: Record> = { "wat", "hoe", "wanneer", "waar", "hoeveel", "asseblief", "kan", "sal", "moet", "wil", "gaan", "dit", "hierdie", "daardie", "nog", "vandag", "môre"]), cy: new Set(["yr", "yn", "gyda", "wrth", "oes", "ydy", "yw", "roedd", "bydd", "fy", "dy", "eich", - "beth", "sut", "pryd", "ble", "pwy", "faint", "dim", "ddim", "ond", "neu", "hefyd", "nawr", - "heddiw", "yfory", "chi", "ti", "nhw", "fe", "gan", "efo"]), + "beth", "sut", "pryd", "ble", "pwy", "faint", "dim", "ddim", "ond", "neu", "hefyd", "nawr", "heddiw", + "yfory", "chi", "ti", "nhw", "fe", "gan", "efo", "yma", "gyfer", "sydd", "dwi", "unrhyw", "wedi", + "gallu", "eisiau", "isio", "fyddai", "rhaid", "chdi", "fo", "hwn", "hon", "hyn", "yna", "rhywbeth", + "pob", "mwy", "iawn", "diolch", "plis"]), + da: new Set(["jeg", "hun", "ham", "hende", "jer", "dine", "hendes", "vores", "jeres", "deres", "og", + "af", "til", "ved", "hos", "før", "mellem", "hvad", "hvem", "hvor", "hvornår", "hvornaar", "hvordan", + "hvorfor", "hvilken", "hvilket", "hvilke", "er", "havde", "kunne", "skal", "ikke", "venligst", + "imorgen", "denne", "dette", "disse", "noget", "nogen", "også", "ogsaa", "meget", "mere", "kun", + "paa"]), + nb: new Set(["jeg", "hun", "dere", "meg", "deg", "ham", "dine", "våre", "deres", "og", "til", "ved", + "hos", "etter", "før", "mellom", "hva", "hvem", "hvor", "når", "hvordan", "hvorfor", "hvilken", + "hvilket", "hvilke", "er", "hadde", "kunne", "skal", "ikke", "nei", "takk", "vennligst", "nå", + "imorgen", "denne", "dette", "disse", "noe", "noen", "også", "mye", "kun"]), + fi: new Set(["minä", "mina", "sinä", "sina", "hän", "minun", "sinun", "hänen", "hanen", "meidän", + "teidän", "heidän", "minulle", "sinulle", "minua", "sinua", "mitä", "mita", "mikä", "mika", "missä", + "missa", "mistä", "mista", "mihin", "milloin", "miten", "kuinka", "kuka", "kenen", "miksi", + "paljonko", "onko", "voitko", "voisitko", "haluan", "haluaisin", "ole", "oli", "ovat", "tai", "mutta", + "että", "etta", "jos", "kun", "myös", "myos", "nyt", "tänään", "tanaan", "huomenna", "kiitos", + "kaikki", "jotain", "joku", "tämä", "tama", "tämän", "taman", "nämä", "nama", "siellä", "siella", + "täällä", "taalla"]), + sl: new Set(["jaz", "ona", "oni", "moj", "moja", "moje", "tvoj", "tvoja", "njegov", "njen", "naš", + "vaš", "vas", "kaj", "kdo", "kje", "kdaj", "kako", "zakaj", "koliko", "kateri", "katera", "katero", + "sem", "smo", "ste", "bo", "bom", "bi", "ker", "če", "z", "iz", "od", "pri", "prosim", "hvala", + "danes", "jutri", "zdaj", "sedaj", "tudi", "samo", "vse", "nekaj"]), }; const NON_EN_DIACRITICS = new Set( diff --git a/laya-ts/tests/undecided-evidence.test.ts b/laya-ts/tests/undecided-evidence.test.ts index e652ce9d3..30d720268 100644 --- a/laya-ts/tests/undecided-evidence.test.ts +++ b/laya-ts/tests/undecided-evidence.test.ts @@ -18,13 +18,26 @@ describe("undecided Latin text with evidence of another language", () => { "huwag mo akong gisingin bukas", // tl, `bukas` "ken jy enige grappe", // af, `jy` "paid a siarad heddiw", // cy, `heddiw` - "schakel de lampen uit", // nl, `uit` + "schakel de lampen uit", // nl, `uit` (Afrikaans too) "vertel me wat er gebeurt op instagram", // nl, `wat` and `op` + "speel mijn rock afspeellijst", // nl, `mijn` + "cine a scris melodia asta", // ro, `cine` + "niambie kuhusu kengele zangu", // sw, `kuhusu` + "vil det regne denne uge", // da, `denne` + "slett alarmen jeg nettopp satte", // nb, `jeg` + "olly kerro minulle vitsi", // fi, `minulle` + "kateri alarmi so nastavljeni", // sl, `kateri` ])("routes to multilingual: %s", (text) => { - expect(guessLatinLanguage(text)).toBeNull(); expect(router.route(text, {}).model).toBe("multilingual"); }); + it.each([ + "streiche alle meine geplanten termine", + "me gustaría escuchar algunos buenos chistes divertidos", + ])("leaves the language undecided: %s", (text) => { + expect(guessLatinLanguage(text)).toBeNull(); + }); + it("names the evidence in the reason", () => { expect(router.route("streiche alle meine geplanten termine", {}).reason) .toContain("function words of other languages"); From c051fb04a28e6ceb6b2f02a1fee3765c869e010f Mon Sep 17 00:00:00 2001 From: Rishit Sharma Date: Mon, 12 Oct 2026 20:32:40 +0530 Subject: [PATCH 9/9] fix(lang): preserve routing guards after rebasing onto main --- laya-ts/tests/undecided-evidence.test.ts | 4 ++-- laya/lang.py | 8 ++++++-- tests/test_hooks_api.py | 10 ++++++++++ tests/test_router.py | 18 +++++++++--------- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/laya-ts/tests/undecided-evidence.test.ts b/laya-ts/tests/undecided-evidence.test.ts index 30d720268..f8922d4e2 100644 --- a/laya-ts/tests/undecided-evidence.test.ts +++ b/laya-ts/tests/undecided-evidence.test.ts @@ -5,7 +5,7 @@ import { FOREIGN_WORDS } from "../src/foreign-words.js"; // Same cases as tests/test_router.py (#54): from four words up, undecided Latin text leaves the default // only with evidence of another language. MASSIVE test utterances unless noted. describe("undecided Latin text with evidence of another language", () => { - const router = new Router(); + const router = new Router({ default: "english" }); it.each([ "streiche alle meine geplanten termine", // de, `alle` and `meine` @@ -56,7 +56,7 @@ describe("undecided Latin text with evidence of another language", () => { "wecke mich auf", // under four words ])("follows the default without it: %s", (text) => { expect(router.route(text, {}).model).toBe("english"); - expect(new Router({ default: "multilingual" }).route(text, {}).model).toBe("multilingual"); + expect(new Router().route(text, {}).model).toBe("multilingual"); }); it("keeps the generated words", () => { diff --git a/laya/lang.py b/laya/lang.py index 0518f6a18..1062babcb 100644 --- a/laya/lang.py +++ b/laya/lang.py @@ -838,6 +838,7 @@ def _analyse_text(text: str) -> Dict[str, object]: "non_latin_fraction": non_latin, "mixed_segment": None} prof_lat = latin_profile(text) lang = prof_lat["language"] + other_evidence = prof_lat["other_evidence"] # The acronym bar belongs here too, not only in the segment scan. The scan runs only once a # state already reads English overall, so a state that is *nothing but* an acronym line -- # or one diluted by fewer English lines than it takes to tip this verdict -- never reached it @@ -859,6 +860,7 @@ def _analyse_text(text: str) -> Dict[str, object]: if not _shouted_evidence(_WORD.findall(text), lang, float(prof_lat["diacritic_rate"])): lang = None + other_evidence = bool(prof_lat["diacritic_rate"]) else: # Mixed-case text: the segment scan has always held that an acronym is not a word, # but the whole-state verdict never applied that rule, so the acronyms voted in it. @@ -885,7 +887,9 @@ def _analyse_text(text: str) -> Dict[str, object]: blanked = _LETTER_RUN.sub( lambda m: " " if m.group().isupper() else m.group(), text) if blanked != text: - lang = latin_profile(blanked)["language"] + blanked_profile = latin_profile(blanked) + lang = blanked_profile["language"] + other_evidence = bool(prof_lat["diacritic_rate"]) or blanked_profile["other_evidence"] # Undecided is not English. Treating it as English sent every Latin-script language we hold no # stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the # language, non-English letters or a shared Swedish-Danish marker can still prefer the @@ -895,7 +899,7 @@ def _analyse_text(text: str) -> Dict[str, object]: # word, still goes to the English one. undecided = lang is None english = lang == "en" or (undecided and not prof_lat["looks_non_english"] - and not prof_lat["other_evidence"]) + and not other_evidence) return {"script": "latin", "script_profile": prof, "language": lang, "is_english": english, "language_undecided": undecided, "diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4), diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py index f1914b7eb..b202badee 100644 --- a/tests/test_hooks_api.py +++ b/tests/test_hooks_api.py @@ -82,6 +82,16 @@ def check_param(name, fn, param, default, kind=None): # Python and TypeScript share the 0.4.0 fallback for unidentified text. check_param("Router.__init__", Router.__init__, "default", "multilingual") +# The evidence rule adds a Latin-profile field while keeping routing detection's public keys. +from laya.lang import latin_profile # noqa: E402 + +for text in ("refund me", "streiche alle meine geplanten termine", "Please refund the duplicate charge"): + check("latin_profile/keys " + text, set(latin_profile(text)), + {"language", "english_hits", "diacritic_rate", "looks_non_english", "other_evidence"}) + check("Router.route/detection keys " + text, set(Router().route(text)["detection"]), + {"script", "script_profile", "language", "is_english", "language_undecided", + "diacritic_rate", "non_latin_fraction", "mixed_segment"}) + # Router keeps lang_guess and explicit per-model revisions too check_param("Router.__init__", Router.__init__, "lang_guess", None) check_param("Router.__init__", Router.__init__, "revisions", None) diff --git a/tests/test_router.py b/tests/test_router.py index 4bdceba07..341320320 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -748,25 +748,25 @@ def _ms(state, reps=9): for text in ["streiche alle meine geplanten termine", # de, `alle` (it) and `meine` (de) "me gustaría escuchar algunos buenos chistes divertidos"]: # es, one `í` check("latin_lang/undecided with evidence " + text, guess_latin_language(text), None) - check("route/undecided with evidence " + text, _r_lat.route(text).model, "multilingual") + check("route/undecided with evidence " + text, _r_en.route(text).model, "multilingual") check("route/undecided evidence reason names the words", "function words of other languages" in - _r_lat.route("streiche alle meine geplanten termine").reason, True) + _r_en.route("streiche alle meine geplanten termine").reason, True) check("route/undecided evidence reason names the letter", "a non-English letter" in - _r_lat.route("me gustaría escuchar algunos buenos chistes divertidos").reason, True) + _r_en.route("me gustaría escuchar algunos buenos chistes divertidos").reason, True) # One word that English uses too is not enough, and neither is one such word twice: `do` and `las` # (both from the English texts checked on #286), so such text follows `default`, Portuguese with it for text in ["how do my health benefits work", "las vegas weather today", "verificar qualquer email da amazon", # pt, `da` "apaga la luz de la cocina"]: # es, `la` twice check("route/one list word follows default " + text, _r_ml.route(text).model, "multilingual") - check("route/one list word stock default " + text, _r_lat.route(text).model, "english") + check("route/one list word english override " + text, _r_en.route(text).model, "english") # Under four words the text is not scored, so two such words are no evidence yet -check("route/undecided three words follow default", _r_lat.route("wecke mich auf").model, "english") -# No evidence, no move: English with no function word stays English (the first three are from the 20,000 +check("route/undecided three words follow default", _r_en.route("wecke mich auf").model, "english") +# With an English default, English with no function word stays English (the first three are from the 20,000 # English texts checked on #286, the last from en-US MASSIVE) ... for text in ["tell me my current savings account's interest rate", "play recently added music", "how many days before my credit card arrives", "turn off lobby light"]: - check("route/english without a function word " + text, _r_lat.route(text).model, "english") + check("route/english without a function word " + text, _r_en.route(text).model, "english") # ... while languages often typed in plain ASCII now have lists, or more words in theirs (Dutch, # Romanian). One word English never uses is enough. MASSIVE test utterances that went to the English # checkpoint before: @@ -787,10 +787,10 @@ def _ms(state, reps=9): "slett alarmen jeg nettopp satte", # nb, `jeg` "olly kerro minulle vitsi", # fi, `minulle` "kateri alarmi so nastavljeni"]: # sl, `kateri` - check("route/new list evidence " + text, _r_lat.route(text).model, "multilingual") + check("route/new list evidence " + text, _r_en.route(text).model, "multilingual") # A word of those lists that English uses too still needs a second one: en-US MASSIVE, `dim` and `kung` for text in ["olly give me some dim light", "kung fu panda three"]: - check("route/english with a new list word " + text, _r_lat.route(text).model, "english") + check("route/english with a new list word " + text, _r_en.route(text).model, "english") # A word leaves _FOREIGN_WORDS with its list, or it would be evidence for no language at all from laya.lang import _FOREIGN_WORDS, _OTHER_WORDS # noqa: E402 check("latin_lang/foreign words all come from a list", sorted(_FOREIGN_WORDS - _OTHER_WORDS), [])