Skip to content
Closed
Prev Previous commit
Next Next commit
feat(lang): count one function word English never uses as evidence of…
… another language

Undecided Latin text of four or more words left the default only with
two distinct list words or a non-English letter. Indonesian, Malay,
Javanese, Swahili, Tagalog, Afrikaans and Welsh had no list, so their
plain ASCII carried no evidence at all and went to the English
checkpoint.

- Add function-word lists for those seven languages. A word another list
  already holds is left out, so no other language's score changes, and so
  is a word in more than 1 in 2,000 English train documents.
- One list word is now evidence on its own when it occurs in none of
  212,592 English documents (train splits of CLINC150, BANKING77, SST-2
  and AG News). Those 568 words are `_FOREIGN_WORDS` in laya/lang.py,
  generated by scripts/build_foreign_words.py (pinned revisions,
  --check), which also writes laya-ts/src/foreign-words.ts. lang.py stays
  self-contained, so research/evals/massive_route_comparison.py can still
  load it from any commit. A word English uses too (`do`, `van`, `per`)
  still needs a second one.
- The route reason says "function words of other languages", since one
  can now be enough.

MASSIVE en-US test: 0 of 2,974 rows move (main: 0). Across 61,830
English texts one moves, the same one as before (`play do re mi`).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
  • Loading branch information
Rish-it and claude committed Oct 12, 2026
commit c2949f58d6faa815626a4d3c4a4c7bb0661bf779
44 changes: 44 additions & 0 deletions laya-ts/src/foreign-words.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/** Function words of other languages that English text does not use.
*
* Generated by `scripts/build_foreign_words.py`; do not edit by hand. Same words as `_FOREIGN_WORDS`
* in `laya/lang.py`: the words of the non-English lists found in none of 212,592 English documents.
* 568 words.
*/
export const FOREIGN_WORDS: ReadonlySet<string> = new Set(`
abar abbiamo aceasta această acest acesta acum adakah adesso agora ainda ajke akan akhon ako aku
algo alguem alguém alla alle alors amader amake amma amra ancaq anche ancora apakah apnake apnar
apnara apni appen aqui aquí arep artiq artıq asbe asseblief atau aterbetala aterbetalning até
auch auf aunque avete awak bado bagaimana bakal behover behöver belum berapa besok bhai bhalo
bien bila bir bisa bitte ble blivit boleh bolte bolun bukan bukas bydd cadê chaina ciao como
consigo cuando daardie dagli daha dalam dalla dallo dapat dari dat ddim debiterade degli dello
dengan depois dessa detta deu deux deve devo devono deyil deze dheweke dhonnobad dich diese
diesen dieser dieses dilam dimana dir dite diye diyechi dois doit donc donde dorkar duibar după
durung də efo eger ei eich ein eine einen einer eita ekbar ekhon ekhono ekta elles endi entao
então esok esta estamos estava esto estou está faktura fel ferot ficou fiz foarte foi fois fost
fra från fue fueron får für fără gaan gan ganger geche gelem gese gibt già gli gostaria gyda
gång gånger gör göra görə habe haben harus heddiw hefyd hemos hendak heute hierdie hii hittar
hiyo hizi hjalp hjalpa hjälp hjälpa hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle huu
här həm hər ich ieri ikaw iki iku ile ilə ingin ini installningarna installningen
inställningarna inställningen inte isso isto isə iyan iyon jabe jetzt jika jodi jonno jours juga
julle jy já kabeh kailan kalau kalke kama kami kamu kanggo kann kannst kapan karena karo katika
keno kepiye kerana kesho keu kharap khub kibhabe kichu kila kintu kita kivabe kokhon kommer
korbo korchi kore koreche korechi korlam korsi korte korun kothay koto kowe kraschar kuwi
kvittot kwa kwamba lagbe lagi lakin lakini lini losenord lui lösenord maar mahu mais mana maneh
mau mayroon meine meinem meinen meiner menyang mereka mes meu meus mimi minha minhas moddhe mois
molto muito mung más måste môre mən naar nach nai naman namin nang nanging nani nao nar natin
nawr necesito negli nel nella neng nenhum nenhuma neu ngapi ngayon nhw nicht nie niet nila ning
ninguem ninguém nini niya niye noch nog noite nossa nosso nous nya nyinyi não när nästa nə
obrigada obrigado och också oder oggi oi olan oleh olmasa olub olá ond onde onek onlar ontem oor
oppnar ora paano pabo pada paini paket paki pakiusap parben parbo parchi parchina pela pelo
pentru perche peux peyechi pira piye più poco pode podem por porque pourquoi preciso pryd puede
pueden pwy până quand quando quero questa questo quiero roedd saan sahaja saja saka sampeyan
sapa sathe saya scusa sedang sekarang sempre semua sesuk shathe shob shomossa siapa sich sila
sisi siya siz skicka skulle sobre somossa sono sonra sparningen spårningen stato sua sudah sul
sulla sulle sunt sut são säger să sən tader tafadhali tahole tambem também tapi tarde tayo telah
tena tengo tenho tetapi theke tidak tiene tienen tolong tomar tomra tous toute trebuie très três
tudo tulung tumi tuo tva två tá ucun uga uit untuk upp uppdaterats utawa valo vandag var veut
veux vill vipi voces você vocês voglio voor vorrei vous vreau vă və waar wae wala wann wanneer
wao wapi welche werden wewe wir wird worden wordt wrth wurde yalniz yalnız ydy yeye yfory yn
yote yox yoxdur yw zijn zum zur är återbetala återbetalning çox è é être în öppnar üçün și ți
əgər
`.split(/\s+/).filter((w) => w.length > 0));
87 changes: 84 additions & 3 deletions laya/lang.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,42 @@
"var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
"hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
"ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"},

# Indonesian, Malay, Javanese, Swahili, Tagalog, Afrikaans and Welsh are mostly typed in plain ASCII,
# so with no list they left no evidence at all and went to the English checkpoint (#54). Two kinds of
# word are left out. A word another list already holds (`di`, `na`, `ni`, `si`, `ya`, `het`, `op`,
# `ar`) would change that language's score. A word found in more than 1 in 2,000 English documents
# (`dan`, `yen`, `au`, `sa`, `mo`, `pa`, `met`, `van`, `os`, `hi`, `mae`, `y`) would be English
# evidence as often as foreign. Indonesian and Malay share most of theirs, so a shared word names
# neither and only `mau`, `mahu`, `kamu` and the like tell them apart.
"id": {"yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "kamu", "anda", "dia",
"kami", "kita", "mereka", "apa", "apakah", "bagaimana", "kapan", "dimana", "mana", "siapa",
"berapa", "tidak", "bukan", "akan", "sudah", "belum", "sedang", "bisa", "dapat", "harus",
"mau", "ingin", "tolong", "juga", "atau", "tetapi", "tapi", "pada", "dalam", "oleh",
"karena", "jika", "kalau", "ada", "semua", "lagi", "saja", "nya", "besok", "sekarang"},
"ms": {"yang", "untuk", "dengan", "dari", "ke", "ini", "itu", "saya", "aku", "awak", "anda", "dia",
"kami", "kita", "mereka", "apa", "adakah", "bagaimana", "bila", "mana", "siapa", "berapa",
"tidak", "tak", "bukan", "akan", "sudah", "telah", "boleh", "mahu", "hendak", "tolong",
"juga", "atau", "tetapi", "pada", "dalam", "oleh", "kerana", "jika", "kalau", "ada",
"semua", "lagi", "sahaja", "esok", "sekarang"},
"jv": {"aku", "kowe", "sampeyan", "dheweke", "iki", "kuwi", "iku", "lan", "karo", "ing", "menyang",
"saka", "kanggo", "apa", "piye", "kepiye", "kapan", "endi", "sapa", "pira", "ora", "wis",
"durung", "arep", "bakal", "isa", "iso", "bisa", "gelem", "tulung", "uga", "utawa",
"nanging", "ana", "kabeh", "maneh", "wae", "mung", "kang", "sing", "ning", "neng", "nang",
"sesuk", "saiki"},
"sw": {"wa", "za", "kwa", "katika", "hii", "hiyo", "huu", "hizi", "mimi", "wewe", "yeye", "sisi",
"nyinyi", "wao", "nini", "vipi", "lini", "wapi", "nani", "ngapi", "sana", "tafadhali",
"kwamba", "lakini", "kama", "bado", "tena", "pia", "yote", "kila", "leo", "kesho"},
"tl": {"ang", "ng", "mga", "ay", "ako", "ikaw", "ka", "siya", "kami", "tayo", "sila", "ko", "niya",
"namin", "natin", "nila", "ito", "iyan", "iyon", "ano", "paano", "kailan", "saan", "sino",
"ilan", "hindi", "oo", "po", "naman", "lang", "rin", "ngayon", "bukas", "kung", "mayroon",
"wala", "gusto", "paki", "pakiusap"},
"af": {"en", "nie", "ek", "jy", "hy", "sy", "ons", "julle", "hulle", "vir", "uit", "oor", "wat",
"hoe", "wanneer", "waar", "hoeveel", "asseblief", "kan", "sal", "moet", "wil", "gaan", "dit",
"hierdie", "daardie", "nog", "vandag", "môre"},
"cy": {"yr", "yn", "gyda", "wrth", "oes", "ydy", "yw", "roedd", "bydd", "fy", "dy", "eich", "beth",
"sut", "pryd", "ble", "pwy", "faint", "dim", "ddim", "ond", "neu", "hefyd", "nawr", "heddiw",
"yfory", "chi", "ti", "nhw", "fe", "gan", "efo"},
}

# Short support fragments often consist of only two or three words, so they do not reach the
Expand Down Expand Up @@ -222,9 +258,53 @@
_EN_COLLISION_WORDS = {"come", "son", "do", "care", "todo", "im", "per", "plus"} & {
w for lg, sw in _STOP.items() if lg != "en" for w in sw}
# Every word a non-English list holds. In text with no English function word, two different ones are
# evidence the text is not English, even when it is too thin to say which language (#54). One is not:
# English uses `do`, `todo` and `van` too ("how do my health benefits work").
# evidence the text is not English, even when it is too thin to say which language (#54). One is not
# when English uses it too: `do`, `todo` and `van` ("how do my health benefits work"). One that
# English text never uses (`saya`, `kowe`, `nicht`) is, and `_FOREIGN_WORDS` holds those.
_OTHER_WORDS = frozenset(w for lg, words in _STOP.items() if lg != "en" for w in words)
# The words of `_OTHER_WORDS` found in none of 212,592 English documents (the train splits of CLINC150,
# BANKING77, SST-2 and AG News). Generated by `scripts/build_foreign_words.py`, which rewrites this set
# and `laya-ts/src/foreign-words.ts`; `--check` says whether they are current. A word added to a list is
# not here until the script runs again, so it needs a second word like `do` does: the safe side.
_FOREIGN_WORDS = frozenset("""
abar abbiamo aceasta această acest acesta acum adakah adesso agora ainda ajke akan akhon ako
aku algo alguem alguém alla alle alors amader amake amma amra ancaq anche ancora apakah
apnake apnar apnara apni appen aqui aquí arep artiq artıq asbe asseblief atau aterbetala
aterbetalning até auch auf aunque avete awak bado bagaimana bakal behover behöver belum
berapa besok bhai bhalo bien bila bir bisa bitte ble blivit boleh bolte bolun bukan bukas
bydd cadê chaina ciao como consigo cuando daardie dagli daha dalam dalla dallo dapat dari
dat ddim debiterade degli dello dengan depois dessa detta deu deux deve devo devono deyil
deze dheweke dhonnobad dich diese diesen dieser dieses dilam dimana dir dite diye diyechi
dois doit donc donde dorkar duibar după durung də efo eger ei eich ein eine einen einer eita
ekbar ekhon ekhono ekta elles endi entao então esok esta estamos estava esto estou está
faktura fel ferot ficou fiz foarte foi fois fost fra från fue fueron får für fără gaan gan
ganger geche gelem gese gibt già gli gostaria gyda gång gånger gör göra görə habe haben
harus heddiw hefyd hemos hendak heute hierdie hii hittar hiyo hizi hjalp hjalpa hjälp hjälpa
hobe hocche hoeveel hoise hoje hoye hoyeche hoyni hulle huu här həm hər ich ieri ikaw iki
iku ile ilə ingin ini installningarna installningen inställningarna inställningen inte isso
isto isə iyan iyon jabe jetzt jika jodi jonno jours juga julle jy já kabeh kailan kalau
kalke kama kami kamu kanggo kann kannst kapan karena karo katika keno kepiye kerana kesho
keu kharap khub kibhabe kichu kila kintu kita kivabe kokhon kommer korbo korchi kore koreche
korechi korlam korsi korte korun kothay koto kowe kraschar kuwi kvittot kwa kwamba lagbe
lagi lakin lakini lini losenord lui lösenord maar mahu mais mana maneh mau mayroon meine
meinem meinen meiner menyang mereka mes meu meus mimi minha minhas moddhe mois molto muito
mung más måste môre mən naar nach nai naman namin nang nanging nani nao nar natin nawr
necesito negli nel nella neng nenhum nenhuma neu ngapi ngayon nhw nicht nie niet nila ning
ninguem ninguém nini niya niye noch nog noite nossa nosso nous nya nyinyi não när nästa nə
obrigada obrigado och också oder oggi oi olan oleh olmasa olub olá ond onde onek onlar ontem
oor oppnar ora paano pabo pada paini paket paki pakiusap parben parbo parchi parchina pela
pelo pentru perche peux peyechi pira piye più poco pode podem por porque pourquoi preciso
pryd puede pueden pwy până quand quando quero questa questo quiero roedd saan sahaja saja
saka sampeyan sapa sathe saya scusa sedang sekarang sempre semua sesuk shathe shob shomossa
siapa sich sila sisi siya siz skicka skulle sobre somossa sono sonra sparningen spårningen
stato sua sudah sul sulla sulle sunt sut são säger să sən tader tafadhali tahole tambem
também tapi tarde tayo telah tena tengo tenho tetapi theke tidak tiene tienen tolong tomar
tomra tous toute trebuie très três tudo tulung tumi tuo tva två tá ucun uga uit untuk upp
uppdaterats utawa valo vandag var veut veux vill vipi voces você vocês voglio voor vorrei
vous vreau vă və waar wae wala wann wanneer wao wapi welche werden wewe wir wird worden
wordt wrth wurde yalniz yalnız ydy yeye yfory yn yote yox yoxdur yw zijn zum zur är
återbetala återbetalning çox è é être în öppnar üçün și ți əgər
""".split())

_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,
Expand Down Expand Up @@ -491,7 +571,8 @@ def latin_profile(text: str) -> Dict[str, object]:
elif en and (not non_english or _english_rescued_by_words(words, diac_rate)):
lang = "en"
# a non-English letter too rare for the rate counts as evidence too
other = diac > 0 or len(_OTHER_WORDS.intersection(words)) >= 2
other = (diac > 0 or len(_OTHER_WORDS.intersection(words)) >= 2
or not _FOREIGN_WORDS.isdisjoint(words))
return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,
"looks_non_english": non_english or (lang is None and nordic_overlap),
"other_evidence": other}
Expand Down
2 changes: 1 addition & 1 deletion laya/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -1280,7 +1280,7 @@ def _route(
# Too few non-English letters to count alone: `analyse` routed it on four or more
# words with no English function word and some evidence of another language (#54).
evidence = ("a non-English letter" if det["diacritic_rate"]
else "two function words of other languages")
else "function words of other languages")
reason = ("Latin script, language not identified; no English function word, but %s; "
"not safe for the English checkpoint" % evidence)
else:
Expand Down
109 changes: 109 additions & 0 deletions scripts/build_foreign_words.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
"""Build the list words that are evidence of another language on their own.

Rewrites ``_FOREIGN_WORDS`` in ``laya/lang.py`` and ``laya-ts/src/foreign-words.ts``: every word
of the non-English stopword lists in ``laya/lang.py`` that occurs in none of the English documents
below. Those are the train splits of four public English corpora, at pinned revisions; their
test splits are where the rule's English cost is measured, so they are not read here.

A word is read the way ``laya.lang`` reads one: a run of letters, lowercased. A list word that
English text does use (``do``, ``van``, ``per``) still needs a second, different one beside it.

Usage::

pip install datasets
python scripts/build_foreign_words.py # download (cached by `datasets`), write
python scripts/build_foreign_words.py --check # exit 1 if the files differ

Run it after changing a list. Until then a new word is not in the set, so it needs a second
word like any other: the safe side for English.
"""

from __future__ import annotations

import argparse
import os
import re
import sys
import textwrap

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PY_OUT = os.path.join(REPO, "laya", "lang.py")
# The set's body in laya/lang.py; everything around it is hand-written and left alone.
PY_BLOCK = re.compile(r'(\n_FOREIGN_WORDS = frozenset\("""\n).*?(""".split\(\)\)\n)', re.S)
TS_OUT = os.path.join(REPO, "laya-ts", "src", "foreign-words.ts")

# (dataset, config, revision, text column), train split of each.
SOURCES = [
("clinc/clinc_oos", "plus", "155b9c710419136e17307b80d0a13e68cd46b4ec", "text"),
("mteb/banking77", None, "18072d2685ea682290f7b8924d94c62acc19c0b2", "text"),
("stanfordnlp/sst2", None, "8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb", "sentence"),
("fancyzhx/ag_news", None, "eb185aade064a813bc0b7f42de02595523103ca4", "text"),
]


def english_vocabulary(word_re):
from datasets import load_dataset

seen, docs = set(), 0
for name, config, revision, column in SOURCES:
for text in load_dataset(name, config, split="train", revision=revision)[column]:
seen.update(w.lower() for w in word_re.findall(text))
docs += 1
return seen, docs


def render_python(words, current):
body = textwrap.fill(" ".join(words), width=96, initial_indent=" ", subsequent_indent=" ")
out, n = PY_BLOCK.subn(lambda m: m.group(1) + body + "\n" + m.group(2), current)
if n != 1:
raise SystemExit("laya/lang.py: no `_FOREIGN_WORDS = frozenset(\"\"\"` block to rewrite")
return out


def render_ts(words, docs):
body = textwrap.fill(" ".join(words), width=96)
return """/** Function words of other languages that English text does not use.
*
* Generated by `scripts/build_foreign_words.py`; do not edit by hand. Same words as `_FOREIGN_WORDS`
* in `laya/lang.py`: the words of the non-English lists found in none of {docs:,} English documents.
* {n} words.
*/
export const FOREIGN_WORDS: ReadonlySet<string> = new Set(`
{body}
`.split(/\\s+/).filter((w) => w.length > 0));
""".format(docs=docs, n=len(words), body=body)


def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("--check", action="store_true", help="compare instead of writing")
args = ap.parse_args(argv)

sys.path.insert(0, REPO)
from laya.lang import _OTHER_WORDS, _WORD

seen, docs = english_vocabulary(_WORD)
words = sorted(_OTHER_WORDS - seen)
stale = []
for path in (PY_OUT, TS_OUT):
current = ""
if os.path.exists(path):
with open(path, encoding="utf-8") as f:
current = f.read()
text = render_python(words, current) if path == PY_OUT else render_ts(words, docs)
if current != text:
stale.append(os.path.relpath(path, REPO))
if not args.check:
with open(path, "w", encoding="utf-8", newline="\n") as f:
f.write(text)
print(f"{len(words)} of {len(_OTHER_WORDS)} list words occur in none of {docs:,} English documents")
if args.check and stale:
print("out of date: " + ", ".join(stale), file=sys.stderr)
return 1
if not args.check:
print("rewrote: " + (", ".join(stale) or "nothing, already current"))
return 0


if __name__ == "__main__":
sys.exit(main())
Loading