Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion laya/lang.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,13 @@
"merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
"alors", "donc"},
"de": {"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
"auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber"},
"auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
"ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine", "meinen",
"meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen", "einem", "einer",
"wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei", "nach", "noch", "bitte",
"heute", "jetzt", "kann", "kannst", "habe", "gibt", "wird",
# shared with English on purpose: counted for English alone, they outvoted short German
"in", "was"},
"es": {"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
"pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
# `de`/`en` are Spanish too, but they are common English tokens as well (`de facto`,
Expand Down
2 changes: 1 addition & 1 deletion tests/test_lang_guess.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ def _load(name):
# `Router().route(...)` cannot fail, because both sides are the same pure call on an
# equally configured router -- a regression would move both together.
BEFORE = [("plain english", "I was charged twice and want a refund", "english"),
("German with umlauts", "Mein Konto wurde zweimal belastet, bitte erstatten Sie", "english"),
("German with umlauts", "Mein Konto wurde zweimal belastet, bitte erstatten Sie", "multilingual"),
("Hindi", "यह एक हिंदी वाक्य है", "multilingual"),
("empty", "", "english"),
("digits", "12345", "english")]
Expand Down
21 changes: 21 additions & 0 deletions tests/test_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,27 @@ def check(name, got, want):
guess_latin_language("Ei nu sunt de acord cu factura, vreau o corecție"), "ro")


# --------------------------------------------------------------------- plain-ASCII German (#54)
# No umlaut for the diacritic rate to catch, and `in`/`was` counted for English alone, so these were
# labelled English and handed to the checkpoint that cannot read them.
for text in ["trage diesen termin in meinen kalender ein",
"wie lautet die temperatur in fulda in hessen",
"schalte das licht im wohnzimmer aus",
"was ist die aktuelle zeit"]:
check("latin_lang/ascii german " + text, guess_latin_language(text), "de")
check("route/ascii german " + text, _r_lat.route(text).model, "multilingual")
check("route/english control for #54",
_r_lat.route("I would like to book a flight to Berlin tomorrow").model, "english")
# English that shares words with the German list stays English. `in` and `den` leave the first, a real
# en-US MASSIVE utterance, one German hit short of flipping; the chat line carries `im` and flips if
# any one of the English words `am`, `an` or `so` joins the German list.
for text in ["turn off smart lamp in den", "im so sorry, am an hour late, stuck in traffic"]:
check("route/english sharing german words " + text, _r_lat.route(text).model, "english")
# German words that Spanish (`es`) or French (`du`) also claim would stop naming those languages
check("latin_lang/spanish es stays evidence", guess_latin_language("que hora es en australia"), "es")
check("latin_lang/french du stays evidence", guess_latin_language("baisse le volume du haut-parleur"), "fr")


# --------------------------------------------------------------------- temperature clamp (#35)
# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which
# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.
Expand Down