From c3ed30ef885faa6a6b241f8c3aa2dd188c721b42 Mon Sep 17 00:00:00 2001 From: Adeshina Lasisi Date: Sun, 4 Oct 2026 19:09:28 +0100 Subject: [PATCH 01/56] feat(java): port laya.lang's script and language detection Routing decides whether a state goes to the English checkpoint or the multilingual one, and the English checkpoint collapses to near-random on a script it has no tokens for (0.100 on Hindi at 20 options, where random is 0.050). So this is the part of the port where a small difference is a wrong answer rather than a rounding discrepancy. What the reference leans on, and what Java gives instead: * four regexes. Java's \w is not Python's -- it admits combining marks and join controls -- and \d and \p{L} follow the JDK's Unicode version rather than CPython's. Each pattern is a hand-written scan over the recorded tables. The identifier scan reproduces the reference's lookbehind, which is what keeps it linear: without it a 4,000-character token costs 205 ms against 0.45 ms for prose, and a state is user input. * str.lower(). Not String.toLowerCase, which applies the contextual final-sigma rule; and not Character.toLowerCase either, which at Unicode 13.0 does not know the lowercase of 40 code points CPython 15.0 maps (all of Vithkuqi among them) and returns them unchanged. A table, so the mapping does not depend on the JDK a consumer runs. * str.isspace(), which strip() and split() work on. Character.isWhitespace rejects four of its members -- U+0085, U+00A0, U+2007, U+202F -- and Character.isSpaceChar is wrong the other way. A no-break space pasted from a word processor otherwise fuses two words into one token that no stopword list holds. * dict insertion order, which is data here: it orders the public script profile and breaks every tie between two languages with the same score. LinkedHashMap throughout, and the fixture now records the stopword order explicitly -- it was being sorted away, which is the same mistake the generator's missing sort_keys was fixed for once already. Faster than the reference and than laya-ts, on the same corpus, best-of-7 after three warmups: Python laya-ts laya-java 1,700 analyse calls 85 ms 34 ms 25 ms 20 calls on a 3.8 KB state 37 ms 14 ms 8 ms Two changes earn most of that, and neither costs a divergence. Script lookup is a binary search over one sorted table instead of a scan over 25 scripts in order -- equivalent because no two ranges overlap, which the generator now refuses to emit a table without, and which a test proves by sweeping every code point. And each character predicate answers ASCII from a pair of 64-bit masks computed from the same predicate as its range table, instead of ten-odd comparisons. Counting letters by script now tallies into an int[] with no boxing or hashing per character. Gated by 137 tests. The corpus is 106 cases recorded from the reference, each asserting all eight output fields and, for a string, all fourteen intermediate steps, so a failure names the stage that broke. The character tables are compared against CPython by a sha256 over all 1,114,112 code points, nine digests in 500 bytes of fixture -- replacing a hand check that proved the tables once and would not have noticed a JDK upgrade moving one. 22 mutants were run against the suite. 19 die. The three survivors are equivalent today for a reason that is now a guarded invariant rather than a coincidence: no script range reaches below U+0370, so the two Latin cuts cannot differ; none of the 40 code points whose case the JDK disputes can start a non-Latin run; and no empty line can be selected, so keeping trailing empty fields on the newline split changes nothing. Each has a test that fails if the enabling property does. The rounding mode needed the same treatment the other way: it is unobservable through analyse at an even threshold, so a half-up mutant passed the whole corpus, and the rule is now pinned by a fixture and the unobservability itself asserted. Also fixes a cliff rather than waiting for it: with every table in its field initialiser the class stopped compiling with "code too large" once the lowercase map was added, because array literals share one and a JVM method body may not exceed 65,535 bytes. Each table now has its own factory method, and the generator refuses to emit one too large for that budget instead of letting javac report it as a size error. 378 tests pass, including the end-to-end parity against the real graph. --- .github/workflows/java.yml | 4 +- laya-java/fixtures/lang_detect.json | 11244 ++++++++++++++++ laya-java/fixtures/lang_tables.json | 13 + .../laya/lang/LanguageDetection.java | 1030 ++ .../laya/lang/LanguageTables.java | 107 +- .../laya/lang/UnicodeTables.java | 1406 +- .../lang/LanguageDetectionParityTest.java | 440 + .../laya/lang/LanguageTablesTest.java | 244 + .../laya/lang/UnicodeTablesTest.java | 204 + laya-java/scripts/gen_fixtures.py | 357 + laya-java/scripts/gen_language_tables.py | 122 +- laya-java/scripts/gen_unicode_tables.py | 271 +- 12 files changed, 14945 insertions(+), 497 deletions(-) create mode 100644 laya-java/fixtures/lang_detect.json create mode 100644 laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java create mode 100644 laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageDetectionParityTest.java create mode 100644 laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageTablesTest.java create mode 100644 laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.java diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 5e6c6cb21..f054bdcb0 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -76,7 +76,7 @@ jobs: if: ${{ !cancelled() }} run: > python laya-java/scripts/check_test_results.py - laya-java/laya-java/build/test-results/test --min-tests 80 --allow-aborted + laya-java/laya-java/build/test-results/test --min-tests 220 --allow-aborted # LAST, deliberately. This gate and the tests are independent signals -- the tests compare # against the COMMITTED fixture, this compares the committed fixture against a fresh @@ -178,7 +178,7 @@ jobs: if: ${{ !cancelled() }} run: > python laya-java/scripts/check_test_results.py - laya-java/laya-java/build/test-results/test --min-tests 200 + laya-java/laya-java/build/test-results/test --min-tests 370 - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ failure() }} diff --git a/laya-java/fixtures/lang_detect.json b/laya-java/fixtures/lang_detect.json new file mode 100644 index 000000000..d8a110a96 --- /dev/null +++ b/laya-java/fixtures/lang_detect.json @@ -0,0 +1,11244 @@ +{ + "notes": "Every output `laya.lang` produces, over a corpus built to break a port.\n\n WHY EACH CASE IS HERE. The corpus is not a sample of real traffic; every entry exercises a\n branch that a reimplementation gets wrong by default:\n\n * the four regexes, which no other language spells the same way -- Java's `\\w` admits\n combining marks that Python's does not, and `String.split` drops trailing empty fields\n where Python's keeps them;\n * `str.lower()`, which is not `String.toLowerCase` (final sigma) and not one-to-one (U+0130);\n * `str.isspace()`, which `Character.isWhitespace` disagrees with on four code points, two of\n them the no-break spaces a pasted ticket is full of;\n * `round(x, 4)` and `round(x)`, both of which are half-to-EVEN on the exact binary value;\n * dict insertion order, which decides the script profile's order and every tie between two\n languages with the same score;\n * and the structured-state scans, where a long English stack trace must not outvote a short\n foreign message, and a name field must not pull an English ticket off the English model.\n\n Each case records the full output of `state_text`, `detect_script`, `script_profile`,\n `latin_profile`, `analyse` and `is_english`. String cases also record every internal helper,\n so a failure names the step that broke rather than only the verdict.\n ", + "thresholds": { + "non_en_diacritic_rate": 0.02, + "english_rescue_diacritic_rate": 0.06, + "non_latin_fraction": 0.2, + "non_latin_min_fraction": 0.1, + "non_latin_min_letters": 10 + }, + "unicode_version": "15.0.0", + "unicode_digests": { + "alpha": "75c91603addca6bd85882ed4ba81c5b5e5ebcc3926b215ffb9572ffd37fb6a1a", + "word": "5c980f4638d051cd70ef4121da029493bf6fb5e87b3cdbe134f403d72fd81228", + "combining": "e2c68a024f6e80a03a953cd09c3ece081f80b7743e8a1da5ca7a2b74a81fe47d", + "digit": "6ec1b4e5fdacf7dab133a768ecdfce5593a6c7c33ce7120ae25af9b781b65705", + "upper": "494a3792b2eb7d335604ad353d2f5db88d1b11178e46de4f1a03e4de080697e7", + "lower": "f3603abf8d8e964512a2905c9d81e74c6e6fc282f2ca86f5052f80717af9be30", + "space": "6c4eda9112d4db3cd23936f071127b489c7a69988e8758a098bb879af25f141e", + "python_lower": "a93175724463d190c5c2bf756ecd8363f3fc3ee72bf93dbe5b6c2c27db80fbaf", + "script_of": "863dbf19c0b480abf327c9d5d85f373da1ab7f7724d2a5fcfb086b63419ead75" + }, + "round_to_int": [ + [ + 0.5, + 0 + ], + [ + 1.5, + 2 + ], + [ + 2.5, + 2 + ], + [ + 3.5, + 4 + ], + [ + 4.5, + 4 + ], + [ + 5.5, + 6 + ], + [ + 6.5, + 6 + ], + [ + 7.5, + 8 + ], + [ + 8.5, + 8 + ], + [ + 9.5, + 10 + ], + [ + 10.5, + 10 + ], + [ + 11.5, + 12 + ], + [ + 12.5, + 12 + ], + [ + 13.5, + 14 + ], + [ + 14.5, + 14 + ], + [ + 15.5, + 16 + ], + [ + 16.5, + 16 + ], + [ + 17.5, + 18 + ], + [ + 18.5, + 18 + ], + [ + 19.5, + 20 + ], + [ + 20.5, + 20 + ], + [ + 21.5, + 22 + ], + [ + 22.5, + 22 + ], + [ + 23.5, + 24 + ], + [ + -0.5, + 0 + ], + [ + -1.5, + -2 + ], + [ + -2.5, + -2 + ], + [ + -3.5, + -4 + ], + [ + -4.5, + -4 + ], + [ + 0.0, + 0 + ], + [ + 1.0, + 1 + ], + [ + 9.0, + 9 + ], + [ + 10.0, + 10 + ], + [ + 0.49999999999999994, + 0 + ], + [ + 9.499999999999998, + 9 + ], + [ + 10.000000000000002, + 10 + ], + [ + 0.1, + 0 + ], + [ + 0.9, + 1 + ], + [ + 1.7000000000000002, + 2 + ], + [ + 2.0, + 2 + ], + [ + 3.3000000000000003, + 3 + ], + [ + 4.0, + 4 + ], + [ + 400.0, + 400 + ], + [ + 0.1234, + 0 + ], + [ + 1.1106, + 1 + ], + [ + 1.234, + 1 + ], + [ + 2.0978, + 2 + ], + [ + 2.468, + 2 + ], + [ + 4.0722, + 4 + ], + [ + 4.936, + 5 + ], + [ + 10.488999999999999, + 10 + ], + [ + 12.34, + 12 + ], + [ + 493.59999999999997, + 494 + ], + [ + 0.2, + 0 + ], + [ + 1.8, + 2 + ], + [ + 3.4000000000000004, + 3 + ], + [ + 6.6000000000000005, + 7 + ], + [ + 8.0, + 8 + ], + [ + 17.0, + 17 + ], + [ + 20.0, + 20 + ], + [ + 800.0, + 800 + ], + [ + 0.25, + 0 + ], + [ + 2.25, + 2 + ], + [ + 4.25, + 4 + ], + [ + 5.0, + 5 + ], + [ + 8.25, + 8 + ], + [ + 21.25, + 21 + ], + [ + 25.0, + 25 + ], + [ + 1000.0, + 1000 + ], + [ + 0.3333, + 0 + ], + [ + 2.9997, + 3 + ], + [ + 3.3329999999999997, + 3 + ], + [ + 5.6661, + 6 + ], + [ + 6.6659999999999995, + 7 + ], + [ + 10.998899999999999, + 11 + ], + [ + 13.331999999999999, + 13 + ], + [ + 28.330499999999997, + 28 + ], + [ + 33.33, + 33 + ], + [ + 1333.2, + 1333 + ], + [ + 42.5, + 42 + ], + [ + 50.0, + 50 + ], + [ + 2000.0, + 2000 + ], + [ + 0.75, + 1 + ], + [ + 6.75, + 7 + ], + [ + 12.75, + 13 + ], + [ + 15.0, + 15 + ], + [ + 24.75, + 25 + ], + [ + 30.0, + 30 + ], + [ + 63.75, + 64 + ], + [ + 75.0, + 75 + ], + [ + 3000.0, + 3000 + ], + [ + 0.9999, + 1 + ], + [ + 8.9991, + 9 + ], + [ + 9.999, + 10 + ], + [ + 16.9983, + 17 + ], + [ + 19.998, + 20 + ], + [ + 32.9967, + 33 + ], + [ + 39.996, + 40 + ], + [ + 84.9915, + 85 + ], + [ + 99.99, + 100 + ], + [ + 3999.6, + 4000 + ], + [ + 0.0312, + 0 + ], + [ + 0.2808, + 0 + ], + [ + 0.312, + 0 + ], + [ + 0.5304, + 1 + ], + [ + 0.624, + 1 + ], + [ + 1.0295999999999998, + 1 + ], + [ + 1.248, + 1 + ], + [ + 2.6519999999999997, + 3 + ], + [ + 3.1199999999999997, + 3 + ], + [ + 124.8, + 125 + ] + ], + "cases": [ + { + "name": "empty", + "state": { + "kind": "string", + "value": "" + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [], + "substitute_identifiers": "", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [], + "blank_upper_runs": "", + "python_lower": "", + "strip": "", + "split_whitespace": [], + "split_newline": [ + "" + ], + "count_alpha": 0, + "code_point_length": 0, + "english_rescued_by_words": false + } + }, + { + "name": "whitespace-only", + "state": { + "kind": "string", + "value": " \t\n " + }, + "state_text": " \t\n ", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [], + "substitute_identifiers": " \t\n ", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [], + "blank_upper_runs": " \t\n ", + "python_lower": " \t\n ", + "strip": "", + "split_whitespace": [], + "split_newline": [ + " \t", + " " + ], + "count_alpha": 0, + "code_point_length": 7, + "english_rescued_by_words": false + } + }, + { + "name": "nbsp-only", + "state": { + "kind": "string", + "value": "  " + }, + "state_text": "  ", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [], + "substitute_identifiers": "  ", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [], + "blank_upper_runs": "  ", + "python_lower": "  ", + "strip": "", + "split_whitespace": [], + "split_newline": [ + "  " + ], + "count_alpha": 0, + "code_point_length": 2, + "english_rescued_by_words": false + } + }, + { + "name": "plain-english", + "state": { + "kind": "string", + "value": "I cannot log in to my account and the password reset email never arrives." + }, + "state_text": "I cannot log in to my account and the password reset email never arrives.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 5, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "I", + "cannot", + "log", + "in", + "to", + "my", + "account", + "and", + "the", + "password", + "reset", + "email", + "never", + "arrives" + ], + "substitute_identifiers": "I cannot log in to my account and the password reset email never arrives.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "I cannot log in to my account and the password reset email never arrives.", + "python_lower": "i cannot log in to my account and the password reset email never arrives.", + "strip": "I cannot log in to my account and the password reset email never arrives.", + "split_whitespace": [ + "I", + "cannot", + "log", + "in", + "to", + "my", + "account", + "and", + "the", + "password", + "reset", + "email", + "never", + "arrives." + ], + "split_newline": [ + "I cannot log in to my account and the password reset email never arrives." + ], + "count_alpha": 59, + "code_point_length": 73, + "english_rescued_by_words": true + } + }, + { + "name": "english-short", + "state": { + "kind": "string", + "value": "reset password" + }, + "state_text": "reset password", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "reset", + "password" + ], + "substitute_identifiers": "reset password", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false + ], + "blank_upper_runs": "reset password", + "python_lower": "reset password", + "strip": "reset password", + "split_whitespace": [ + "reset", + "password" + ], + "split_newline": [ + "reset password" + ], + "count_alpha": 13, + "code_point_length": 14, + "english_rescued_by_words": false + } + }, + { + "name": "english-one-word", + "state": { + "kind": "string", + "value": "refund" + }, + "state_text": "refund", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "refund" + ], + "substitute_identifiers": "refund", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "refund", + "python_lower": "refund", + "strip": "refund", + "split_whitespace": [ + "refund" + ], + "split_newline": [ + "refund" + ], + "count_alpha": 6, + "code_point_length": 6, + "english_rescued_by_words": false + } + }, + { + "name": "english-three-words", + "state": { + "kind": "string", + "value": "please reset this" + }, + "state_text": "please reset this", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "please", + "reset", + "this" + ], + "substitute_identifiers": "please reset this", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false + ], + "blank_upper_runs": "please reset this", + "python_lower": "please reset this", + "strip": "please reset this", + "split_whitespace": [ + "please", + "reset", + "this" + ], + "split_newline": [ + "please reset this" + ], + "count_alpha": 15, + "code_point_length": 17, + "english_rescued_by_words": true + } + }, + { + "name": "english-with-cafe", + "state": { + "kind": "string", + "value": "We met at the cafe near the office and the invoice was paid." + }, + "state_text": "We met at the cafe near the office and the invoice was paid.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 7, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "We", + "met", + "at", + "the", + "cafe", + "near", + "the", + "office", + "and", + "the", + "invoice", + "was", + "paid" + ], + "substitute_identifiers": "We met at the cafe near the office and the invoice was paid.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "We met at the cafe near the office and the invoice was paid.", + "python_lower": "we met at the cafe near the office and the invoice was paid.", + "strip": "We met at the cafe near the office and the invoice was paid.", + "split_whitespace": [ + "We", + "met", + "at", + "the", + "cafe", + "near", + "the", + "office", + "and", + "the", + "invoice", + "was", + "paid." + ], + "split_newline": [ + "We met at the cafe near the office and the invoice was paid." + ], + "count_alpha": 47, + "code_point_length": 60, + "english_rescued_by_words": true + } + }, + { + "name": "english-with-accented-loanword", + "state": { + "kind": "string", + "value": "We met at the café near the office and the invoice was paid." + }, + "state_text": "We met at the café near the office and the invoice was paid.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 7, + "diacritic_rate": 0.016666666666666666, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0167, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "We", + "met", + "at", + "the", + "café", + "near", + "the", + "office", + "and", + "the", + "invoice", + "was", + "paid" + ], + "substitute_identifiers": "We met at the café near the office and the invoice was paid.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "We met at the café near the office and the invoice was paid.", + "python_lower": "we met at the café near the office and the invoice was paid.", + "strip": "We met at the café near the office and the invoice was paid.", + "split_whitespace": [ + "We", + "met", + "at", + "the", + "café", + "near", + "the", + "office", + "and", + "the", + "invoice", + "was", + "paid." + ], + "split_newline": [ + "We met at the café near the office and the invoice was paid." + ], + "count_alpha": 47, + "code_point_length": 60, + "english_rescued_by_words": true + } + }, + { + "name": "english-with-three-accents", + "state": { + "kind": "string", + "value": "The café résumé of José was attached to the ticket." + }, + "state_text": "The café résumé of José was attached to the ticket.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 5, + "diacritic_rate": 0.0784313725490196, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0784, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "The", + "café", + "résumé", + "of", + "José", + "was", + "attached", + "to", + "the", + "ticket" + ], + "substitute_identifiers": "The café résumé of José was attached to the ticket.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The café résumé of José was attached to the ticket.", + "python_lower": "the café résumé of josé was attached to the ticket.", + "strip": "The café résumé of José was attached to the ticket.", + "split_whitespace": [ + "The", + "café", + "résumé", + "of", + "José", + "was", + "attached", + "to", + "the", + "ticket." + ], + "split_newline": [ + "The café résumé of José was attached to the ticket." + ], + "count_alpha": 41, + "code_point_length": 51, + "english_rescued_by_words": false + } + }, + { + "name": "english-rescue-two-en-only-words", + "state": { + "kind": "string", + "value": "Please the invoice café and we will have it." + }, + "state_text": "Please the invoice café and we will have it.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 7, + "diacritic_rate": 0.022727272727272728, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0227, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "Please", + "the", + "invoice", + "café", + "and", + "we", + "will", + "have", + "it" + ], + "substitute_identifiers": "Please the invoice café and we will have it.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Please the invoice café and we will have it.", + "python_lower": "please the invoice café and we will have it.", + "strip": "Please the invoice café and we will have it.", + "split_whitespace": [ + "Please", + "the", + "invoice", + "café", + "and", + "we", + "will", + "have", + "it." + ], + "split_newline": [ + "Please the invoice café and we will have it." + ], + "count_alpha": 35, + "code_point_length": 44, + "english_rescued_by_words": true + } + }, + { + "name": "french", + "state": { + "kind": "string", + "value": "Je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas." + }, + "state_text": "Je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "fr", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "fr", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Je", + "ne", + "peux", + "pas", + "me", + "connecter", + "a", + "mon", + "compte", + "et", + "le", + "mot", + "de", + "passe", + "ne", + "fonctionne", + "pas" + ], + "substitute_identifiers": "Je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas.", + "non_latin_words": [], + "named_prose_language": "fr", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas.", + "python_lower": "je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas.", + "strip": "Je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas.", + "split_whitespace": [ + "Je", + "ne", + "peux", + "pas", + "me", + "connecter", + "a", + "mon", + "compte", + "et", + "le", + "mot", + "de", + "passe", + "ne", + "fonctionne", + "pas." + ], + "split_newline": [ + "Je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas." + ], + "count_alpha": 61, + "code_point_length": 78, + "english_rescued_by_words": false + } + }, + { + "name": "french-accented", + "state": { + "kind": "string", + "value": "Je n'arrive pas à me connecter à mon compte, le mot de passe est refusé." + }, + "state_text": "Je n'arrive pas à me connecter à mon compte, le mot de passe est refusé.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "fr", + "english_hits": 0, + "diacritic_rate": 0.041666666666666664, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "fr", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0417, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Je", + "n", + "arrive", + "pas", + "à", + "me", + "connecter", + "à", + "mon", + "compte", + "le", + "mot", + "de", + "passe", + "est", + "refusé" + ], + "substitute_identifiers": "Je n'arrive pas à me connecter à mon compte, le mot de passe est refusé.", + "non_latin_words": [], + "named_prose_language": "fr", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Je n'arrive pas à me connecter à mon compte, le mot de passe est refusé.", + "python_lower": "je n'arrive pas à me connecter à mon compte, le mot de passe est refusé.", + "strip": "Je n'arrive pas à me connecter à mon compte, le mot de passe est refusé.", + "split_whitespace": [ + "Je", + "n'arrive", + "pas", + "à", + "me", + "connecter", + "à", + "mon", + "compte,", + "le", + "mot", + "de", + "passe", + "est", + "refusé." + ], + "split_newline": [ + "Je n'arrive pas à me connecter à mon compte, le mot de passe est refusé." + ], + "count_alpha": 55, + "code_point_length": 72, + "english_rescued_by_words": false + } + }, + { + "name": "spanish", + "state": { + "kind": "string", + "value": "No puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada." + }, + "state_text": "No puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "es", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "es", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "No", + "puedo", + "iniciar", + "sesion", + "en", + "mi", + "cuenta", + "y", + "la", + "contrasena", + "no", + "funciona", + "para", + "nada" + ], + "substitute_identifiers": "No puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada.", + "non_latin_words": [], + "named_prose_language": "es", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "No puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada.", + "python_lower": "no puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada.", + "strip": "No puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada.", + "split_whitespace": [ + "No", + "puedo", + "iniciar", + "sesion", + "en", + "mi", + "cuenta", + "y", + "la", + "contrasena", + "no", + "funciona", + "para", + "nada." + ], + "split_newline": [ + "No puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada." + ], + "count_alpha": 61, + "code_point_length": 75, + "english_rescued_by_words": false + } + }, + { + "name": "spanish-accented", + "state": { + "kind": "string", + "value": "No puedo iniciar sesión en mi cuenta y la contraseña no funciona." + }, + "state_text": "No puedo iniciar sesión en mi cuenta y la contraseña no funciona.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "es", + "english_hits": 0, + "diacritic_rate": 0.03076923076923077, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "es", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0308, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "No", + "puedo", + "iniciar", + "sesión", + "en", + "mi", + "cuenta", + "y", + "la", + "contraseña", + "no", + "funciona" + ], + "substitute_identifiers": "No puedo iniciar sesión en mi cuenta y la contraseña no funciona.", + "non_latin_words": [], + "named_prose_language": "es", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "No puedo iniciar sesión en mi cuenta y la contraseña no funciona.", + "python_lower": "no puedo iniciar sesión en mi cuenta y la contraseña no funciona.", + "strip": "No puedo iniciar sesión en mi cuenta y la contraseña no funciona.", + "split_whitespace": [ + "No", + "puedo", + "iniciar", + "sesión", + "en", + "mi", + "cuenta", + "y", + "la", + "contraseña", + "no", + "funciona." + ], + "split_newline": [ + "No puedo iniciar sesión en mi cuenta y la contraseña no funciona." + ], + "count_alpha": 53, + "code_point_length": 65, + "english_rescued_by_words": false + } + }, + { + "name": "portuguese-stripped", + "state": { + "kind": "string", + "value": "Voce pode me mandar a nota fiscal do pedido que eu fiz ontem?" + }, + "state_text": "Voce pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "pt", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Voce", + "pode", + "me", + "mandar", + "a", + "nota", + "fiscal", + "do", + "pedido", + "que", + "eu", + "fiz", + "ontem" + ], + "substitute_identifiers": "Voce pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "non_latin_words": [], + "named_prose_language": "pt", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Voce pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "python_lower": "voce pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "strip": "Voce pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "split_whitespace": [ + "Voce", + "pode", + "me", + "mandar", + "a", + "nota", + "fiscal", + "do", + "pedido", + "que", + "eu", + "fiz", + "ontem?" + ], + "split_newline": [ + "Voce pode me mandar a nota fiscal do pedido que eu fiz ontem?" + ], + "count_alpha": 48, + "code_point_length": 61, + "english_rescued_by_words": false + } + }, + { + "name": "portuguese-accented", + "state": { + "kind": "string", + "value": "Você pode me mandar a nota fiscal do pedido que eu fiz ontem?" + }, + "state_text": "Você pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "pt", + "english_hits": 0, + "diacritic_rate": 0.01639344262295082, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0164, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Você", + "pode", + "me", + "mandar", + "a", + "nota", + "fiscal", + "do", + "pedido", + "que", + "eu", + "fiz", + "ontem" + ], + "substitute_identifiers": "Você pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "non_latin_words": [], + "named_prose_language": "pt", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Você pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "python_lower": "você pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "strip": "Você pode me mandar a nota fiscal do pedido que eu fiz ontem?", + "split_whitespace": [ + "Você", + "pode", + "me", + "mandar", + "a", + "nota", + "fiscal", + "do", + "pedido", + "que", + "eu", + "fiz", + "ontem?" + ], + "split_newline": [ + "Você pode me mandar a nota fiscal do pedido que eu fiz ontem?" + ], + "count_alpha": 48, + "code_point_length": 61, + "english_rescued_by_words": false + } + }, + { + "name": "portuguese-jargon-english", + "state": { + "kind": "string", + "value": "Deu erro 500 no endpoint de login depois do update de ontem" + }, + "state_text": "Deu erro 500 no endpoint de login depois do update de ontem", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "pt", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Deu", + "erro", + "no", + "endpoint", + "de", + "login", + "depois", + "do", + "update", + "de", + "ontem" + ], + "substitute_identifiers": "Deu erro 500 no endpoint de login depois do update de ontem", + "non_latin_words": [], + "named_prose_language": "pt", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Deu erro 500 no endpoint de login depois do update de ontem", + "python_lower": "deu erro 500 no endpoint de login depois do update de ontem", + "strip": "Deu erro 500 no endpoint de login depois do update de ontem", + "split_whitespace": [ + "Deu", + "erro", + "500", + "no", + "endpoint", + "de", + "login", + "depois", + "do", + "update", + "de", + "ontem" + ], + "split_newline": [ + "Deu erro 500 no endpoint de login depois do update de ontem" + ], + "count_alpha": 45, + "code_point_length": 59, + "english_rescued_by_words": false + } + }, + { + "name": "italian", + "state": { + "kind": "string", + "value": "Non riesco ad accedere al mio account e la password non funziona piu" + }, + "state_text": "Non riesco ad accedere al mio account e la password non funziona piu", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "it", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "it", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Non", + "riesco", + "ad", + "accedere", + "al", + "mio", + "account", + "e", + "la", + "password", + "non", + "funziona", + "piu" + ], + "substitute_identifiers": "Non riesco ad accedere al mio account e la password non funziona piu", + "non_latin_words": [], + "named_prose_language": "it", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Non riesco ad accedere al mio account e la password non funziona piu", + "python_lower": "non riesco ad accedere al mio account e la password non funziona piu", + "strip": "Non riesco ad accedere al mio account e la password non funziona piu", + "split_whitespace": [ + "Non", + "riesco", + "ad", + "accedere", + "al", + "mio", + "account", + "e", + "la", + "password", + "non", + "funziona", + "piu" + ], + "split_newline": [ + "Non riesco ad accedere al mio account e la password non funziona piu" + ], + "count_alpha": 56, + "code_point_length": 68, + "english_rescued_by_words": false + } + }, + { + "name": "italian-articulated", + "state": { + "kind": "string", + "value": "Ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento" + }, + "state_text": "Ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "it", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "it", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Ho", + "ricevuto", + "la", + "fattura", + "nel", + "mese", + "di", + "marzo", + "ma", + "non", + "trovo", + "il", + "pagamento" + ], + "substitute_identifiers": "Ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento", + "non_latin_words": [], + "named_prose_language": "it", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento", + "python_lower": "ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento", + "strip": "Ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento", + "split_whitespace": [ + "Ho", + "ricevuto", + "la", + "fattura", + "nel", + "mese", + "di", + "marzo", + "ma", + "non", + "trovo", + "il", + "pagamento" + ], + "split_newline": [ + "Ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento" + ], + "count_alpha": 54, + "code_point_length": 66, + "english_rescued_by_words": false + } + }, + { + "name": "german", + "state": { + "kind": "string", + "value": "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert." + }, + "state_text": "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "de", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "de", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Ich", + "kann", + "mich", + "nicht", + "in", + "mein", + "Konto", + "einloggen", + "und", + "das", + "Passwort", + "wird", + "nicht", + "akzeptiert" + ], + "substitute_identifiers": "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert.", + "non_latin_words": [], + "named_prose_language": "de", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert.", + "python_lower": "ich kann mich nicht in mein konto einloggen und das passwort wird nicht akzeptiert.", + "strip": "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert.", + "split_whitespace": [ + "Ich", + "kann", + "mich", + "nicht", + "in", + "mein", + "Konto", + "einloggen", + "und", + "das", + "Passwort", + "wird", + "nicht", + "akzeptiert." + ], + "split_newline": [ + "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert." + ], + "count_alpha": 69, + "code_point_length": 83, + "english_rescued_by_words": false + } + }, + { + "name": "german-der-twice", + "state": { + "kind": "string", + "value": "reduzieren der helligkeit der lichter" + }, + "state_text": "reduzieren der helligkeit der lichter", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "de", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "de", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "reduzieren", + "der", + "helligkeit", + "der", + "lichter" + ], + "substitute_identifiers": "reduzieren der helligkeit der lichter", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false + ], + "blank_upper_runs": "reduzieren der helligkeit der lichter", + "python_lower": "reduzieren der helligkeit der lichter", + "strip": "reduzieren der helligkeit der lichter", + "split_whitespace": [ + "reduzieren", + "der", + "helligkeit", + "der", + "lichter" + ], + "split_newline": [ + "reduzieren der helligkeit der lichter" + ], + "count_alpha": 33, + "code_point_length": 37, + "english_rescued_by_words": false + } + }, + { + "name": "dutch", + "state": { + "kind": "string", + "value": "Het is niet mogelijk om in te loggen op mijn account met dit wachtwoord" + }, + "state_text": "Het is niet mogelijk om in te loggen op mijn account met dit wachtwoord", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "nl", + "english_hits": 2, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "nl", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Het", + "is", + "niet", + "mogelijk", + "om", + "in", + "te", + "loggen", + "op", + "mijn", + "account", + "met", + "dit", + "wachtwoord" + ], + "substitute_identifiers": "Het is niet mogelijk om in te loggen op mijn account met dit wachtwoord", + "non_latin_words": [], + "named_prose_language": "nl", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Het is niet mogelijk om in te loggen op mijn account met dit wachtwoord", + "python_lower": "het is niet mogelijk om in te loggen op mijn account met dit wachtwoord", + "strip": "Het is niet mogelijk om in te loggen op mijn account met dit wachtwoord", + "split_whitespace": [ + "Het", + "is", + "niet", + "mogelijk", + "om", + "in", + "te", + "loggen", + "op", + "mijn", + "account", + "met", + "dit", + "wachtwoord" + ], + "split_newline": [ + "Het is niet mogelijk om in te loggen op mijn account met dit wachtwoord" + ], + "count_alpha": 58, + "code_point_length": 71, + "english_rescued_by_words": false + } + }, + { + "name": "swedish", + "state": { + "kind": "string", + "value": "Jag kan inte logga in pa mitt konto och jag behover hjalp med losenord" + }, + "state_text": "Jag kan inte logga in pa mitt konto och jag behover hjalp med losenord", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "sv", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "sv", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Jag", + "kan", + "inte", + "logga", + "in", + "pa", + "mitt", + "konto", + "och", + "jag", + "behover", + "hjalp", + "med", + "losenord" + ], + "substitute_identifiers": "Jag kan inte logga in pa mitt konto och jag behover hjalp med losenord", + "non_latin_words": [], + "named_prose_language": "sv", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Jag kan inte logga in pa mitt konto och jag behover hjalp med losenord", + "python_lower": "jag kan inte logga in pa mitt konto och jag behover hjalp med losenord", + "strip": "Jag kan inte logga in pa mitt konto och jag behover hjalp med losenord", + "split_whitespace": [ + "Jag", + "kan", + "inte", + "logga", + "in", + "pa", + "mitt", + "konto", + "och", + "jag", + "behover", + "hjalp", + "med", + "losenord" + ], + "split_newline": [ + "Jag kan inte logga in pa mitt konto och jag behover hjalp med losenord" + ], + "count_alpha": 57, + "code_point_length": 70, + "english_rescued_by_words": false + } + }, + { + "name": "swedish-login-phrase", + "state": { + "kind": "string", + "value": "kan inte logga in" + }, + "state_text": "kan inte logga in", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "sv", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "sv", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "kan", + "inte", + "logga", + "in" + ], + "substitute_identifiers": "kan inte logga in", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "kan inte logga in", + "python_lower": "kan inte logga in", + "strip": "kan inte logga in", + "split_whitespace": [ + "kan", + "inte", + "logga", + "in" + ], + "split_newline": [ + "kan inte logga in" + ], + "count_alpha": 14, + "code_point_length": 17, + "english_rescued_by_words": false + } + }, + { + "name": "swedish-short-fragment", + "state": { + "kind": "string", + "value": "glomt losenord" + }, + "state_text": "glomt losenord", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "sv", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "sv", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "glomt", + "losenord" + ], + "substitute_identifiers": "glomt losenord", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false + ], + "blank_upper_runs": "glomt losenord", + "python_lower": "glomt losenord", + "strip": "glomt losenord", + "split_whitespace": [ + "glomt", + "losenord" + ], + "split_newline": [ + "glomt losenord" + ], + "count_alpha": 13, + "code_point_length": 14, + "english_rescued_by_words": false + } + }, + { + "name": "swedish-one-short-word", + "state": { + "kind": "string", + "value": "losenord" + }, + "state_text": "losenord", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "losenord" + ], + "substitute_identifiers": "losenord", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "losenord", + "python_lower": "losenord", + "strip": "losenord", + "split_whitespace": [ + "losenord" + ], + "split_newline": [ + "losenord" + ], + "count_alpha": 8, + "code_point_length": 8, + "english_rescued_by_words": false + } + }, + { + "name": "danish-nordic-overlap", + "state": { + "kind": "string", + "value": "Hej jeg kan ikke komme ind pa min konto" + }, + "state_text": "Hej jeg kan ikke komme ind pa min konto", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Hej", + "jeg", + "kan", + "ikke", + "komme", + "ind", + "pa", + "min", + "konto" + ], + "substitute_identifiers": "Hej jeg kan ikke komme ind pa min konto", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Hej jeg kan ikke komme ind pa min konto", + "python_lower": "hej jeg kan ikke komme ind pa min konto", + "strip": "Hej jeg kan ikke komme ind pa min konto", + "split_whitespace": [ + "Hej", + "jeg", + "kan", + "ikke", + "komme", + "ind", + "pa", + "min", + "konto" + ], + "split_newline": [ + "Hej jeg kan ikke komme ind pa min konto" + ], + "count_alpha": 31, + "code_point_length": 39, + "english_rescued_by_words": false + } + }, + { + "name": "romanian", + "state": { + "kind": "string", + "value": "Nu pot sa intru in contul meu si parola nu functioneaza pentru care am" + }, + "state_text": "Nu pot sa intru in contul meu si parola nu functioneaza pentru care am", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "ro", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "ro", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Nu", + "pot", + "sa", + "intru", + "in", + "contul", + "meu", + "si", + "parola", + "nu", + "functioneaza", + "pentru", + "care", + "am" + ], + "substitute_identifiers": "Nu pot sa intru in contul meu si parola nu functioneaza pentru care am", + "non_latin_words": [], + "named_prose_language": "ro", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Nu pot sa intru in contul meu si parola nu functioneaza pentru care am", + "python_lower": "nu pot sa intru in contul meu si parola nu functioneaza pentru care am", + "strip": "Nu pot sa intru in contul meu si parola nu functioneaza pentru care am", + "split_whitespace": [ + "Nu", + "pot", + "sa", + "intru", + "in", + "contul", + "meu", + "si", + "parola", + "nu", + "functioneaza", + "pentru", + "care", + "am" + ], + "split_newline": [ + "Nu pot sa intru in contul meu si parola nu functioneaza pentru care am" + ], + "count_alpha": 57, + "code_point_length": 70, + "english_rescued_by_words": false + } + }, + { + "name": "romanian-diacritics", + "state": { + "kind": "string", + "value": "Nu pot să intru în contul meu și parola nu funcționează" + }, + "state_text": "Nu pot să intru în contul meu și parola nu funcționează", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "ro", + "english_hits": 0, + "diacritic_rate": 0.09090909090909091, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "ro", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0909, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Nu", + "pot", + "să", + "intru", + "în", + "contul", + "meu", + "și", + "parola", + "nu", + "funcționează" + ], + "substitute_identifiers": "Nu pot să intru în contul meu și parola nu funcționează", + "non_latin_words": [], + "named_prose_language": "ro", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Nu pot să intru în contul meu și parola nu funcționează", + "python_lower": "nu pot să intru în contul meu și parola nu funcționează", + "strip": "Nu pot să intru în contul meu și parola nu funcționează", + "split_whitespace": [ + "Nu", + "pot", + "să", + "intru", + "în", + "contul", + "meu", + "și", + "parola", + "nu", + "funcționează" + ], + "split_newline": [ + "Nu pot să intru în contul meu și parola nu funcționează" + ], + "count_alpha": 45, + "code_point_length": 55, + "english_rescued_by_words": false + } + }, + { + "name": "banglish", + "state": { + "kind": "string", + "value": "ami amar account e login korte parchi na, password ta kaj korche na" + }, + "state_text": "ami amar account e login korte parchi na, password ta kaj korche na", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "bn", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "bn", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "ami", + "amar", + "account", + "e", + "login", + "korte", + "parchi", + "na", + "password", + "ta", + "kaj", + "korche", + "na" + ], + "substitute_identifiers": "ami amar account e login korte parchi na, password ta kaj korche na", + "non_latin_words": [], + "named_prose_language": "bn", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "ami amar account e login korte parchi na, password ta kaj korche na", + "python_lower": "ami amar account e login korte parchi na, password ta kaj korche na", + "strip": "ami amar account e login korte parchi na, password ta kaj korche na", + "split_whitespace": [ + "ami", + "amar", + "account", + "e", + "login", + "korte", + "parchi", + "na,", + "password", + "ta", + "kaj", + "korche", + "na" + ], + "split_newline": [ + "ami amar account e login korte parchi na, password ta kaj korche na" + ], + "count_alpha": 54, + "code_point_length": 67, + "english_rescued_by_words": false + } + }, + { + "name": "azerbaijani", + "state": { + "kind": "string", + "value": "Mən hesabima daxil ola bilmirəm və şifrə işləmir, bu üçün kömək" + }, + "state_text": "Mən hesabima daxil ola bilmirəm və şifrə işləmir, bu üçün kömək", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "az", + "english_hits": 0, + "diacritic_rate": 0.19047619047619047, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "az", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.1905, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Mən", + "hesabima", + "daxil", + "ola", + "bilmirəm", + "və", + "şifrə", + "işləmir", + "bu", + "üçün", + "kömək" + ], + "substitute_identifiers": "Mən hesabima daxil ola bilmirəm və şifrə işləmir, bu üçün kömək", + "non_latin_words": [], + "named_prose_language": "az", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Mən hesabima daxil ola bilmirəm və şifrə işləmir, bu üçün kömək", + "python_lower": "mən hesabima daxil ola bilmirəm və şifrə işləmir, bu üçün kömək", + "strip": "Mən hesabima daxil ola bilmirəm və şifrə işləmir, bu üçün kömək", + "split_whitespace": [ + "Mən", + "hesabima", + "daxil", + "ola", + "bilmirəm", + "və", + "şifrə", + "işləmir,", + "bu", + "üçün", + "kömək" + ], + "split_newline": [ + "Mən hesabima daxil ola bilmirəm və şifrə işləmir, bu üçün kömək" + ], + "count_alpha": 52, + "code_point_length": 63, + "english_rescued_by_words": false + } + }, + { + "name": "polish-no-stoplist", + "state": { + "kind": "string", + "value": "Nie mogę się zalogować na swoje konto, hasło nie działa" + }, + "state_text": "Nie mogę się zalogować na swoje konto, hasło nie działa", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.09090909090909091, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0909, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Nie", + "mogę", + "się", + "zalogować", + "na", + "swoje", + "konto", + "hasło", + "nie", + "działa" + ], + "substitute_identifiers": "Nie mogę się zalogować na swoje konto, hasło nie działa", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Nie mogę się zalogować na swoje konto, hasło nie działa", + "python_lower": "nie mogę się zalogować na swoje konto, hasło nie działa", + "strip": "Nie mogę się zalogować na swoje konto, hasło nie działa", + "split_whitespace": [ + "Nie", + "mogę", + "się", + "zalogować", + "na", + "swoje", + "konto,", + "hasło", + "nie", + "działa" + ], + "split_newline": [ + "Nie mogę się zalogować na swoje konto, hasło nie działa" + ], + "count_alpha": 45, + "code_point_length": 55, + "english_rescued_by_words": false + } + }, + { + "name": "turkish-no-stoplist", + "state": { + "kind": "string", + "value": "Hesabıma giriş yapamıyorum ve şifre çalışmıyor, yardım" + }, + "state_text": "Hesabıma giriş yapamıyorum ve şifre çalışmıyor, yardım", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.16666666666666666, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.1667, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Hesabıma", + "giriş", + "yapamıyorum", + "ve", + "şifre", + "çalışmıyor", + "yardım" + ], + "substitute_identifiers": "Hesabıma giriş yapamıyorum ve şifre çalışmıyor, yardım", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Hesabıma giriş yapamıyorum ve şifre çalışmıyor, yardım", + "python_lower": "hesabıma giriş yapamıyorum ve şifre çalışmıyor, yardım", + "strip": "Hesabıma giriş yapamıyorum ve şifre çalışmıyor, yardım", + "split_whitespace": [ + "Hesabıma", + "giriş", + "yapamıyorum", + "ve", + "şifre", + "çalışmıyor,", + "yardım" + ], + "split_newline": [ + "Hesabıma giriş yapamıyorum ve şifre çalışmıyor, yardım" + ], + "count_alpha": 47, + "code_point_length": 54, + "english_rescued_by_words": false + } + }, + { + "name": "turkish-dotted-capital-I", + "state": { + "kind": "string", + "value": "İstanbul İzmir İngilizce" + }, + "state_text": "İstanbul İzmir İngilizce", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "İstanbul", + "İzmir", + "İngilizce" + ], + "substitute_identifiers": "İstanbul İzmir İngilizce", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false + ], + "blank_upper_runs": "İstanbul İzmir İngilizce", + "python_lower": "i̇stanbul i̇zmir i̇ngilizce", + "strip": "İstanbul İzmir İngilizce", + "split_whitespace": [ + "İstanbul", + "İzmir", + "İngilizce" + ], + "split_newline": [ + "İstanbul İzmir İngilizce" + ], + "count_alpha": 22, + "code_point_length": 24, + "english_rescued_by_words": false + } + }, + { + "name": "shared-words-only", + "state": { + "kind": "string", + "value": "la o un de pe ca e que" + }, + "state_text": "la o un de pe ca e que", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "la", + "o", + "un", + "de", + "pe", + "ca", + "e", + "que" + ], + "substitute_identifiers": "la o un de pe ca e que", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "la o un de pe ca e que", + "python_lower": "la o un de pe ca e que", + "strip": "la o un de pe ca e que", + "split_whitespace": [ + "la", + "o", + "un", + "de", + "pe", + "ca", + "e", + "que" + ], + "split_newline": [ + "la o un de pe ca e que" + ], + "count_alpha": 15, + "code_point_length": 22, + "english_rescued_by_words": false + } + }, + { + "name": "hindi", + "state": { + "kind": "string", + "value": "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूं" + }, + "state_text": "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूं", + "detect_script": "devanagari", + "script_profile": { + "devanagari": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "devanagari", + "script_profile": { + "devanagari": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "म", + "अपन", + "ख", + "त", + "म", + "ल", + "ग", + "इन", + "नह", + "कर", + "प", + "रह", + "ह" + ], + "substitute_identifiers": "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूं", + "non_latin_words": [ + "मैं", + "अपने", + "खाते", + "में", + "लॉग", + "इन", + "नहीं", + "कर", + "पा", + "रहा", + "हूं" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूं", + "python_lower": "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूं", + "strip": "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूं", + "split_whitespace": [ + "मैं", + "अपने", + "खाते", + "में", + "लॉग", + "इन", + "नहीं", + "कर", + "पा", + "रहा", + "हूं" + ], + "split_newline": [ + "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूं" + ], + "count_alpha": 19, + "code_point_length": 43, + "english_rescued_by_words": false + } + }, + { + "name": "korean", + "state": { + "kind": "string", + "value": "계정에 로그인할 수 없어요" + }, + "state_text": "계정에 로그인할 수 없어요", + "detect_script": "hangul", + "script_profile": { + "hangul": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "hangul", + "script_profile": { + "hangul": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "계정에", + "로그인할", + "수", + "없어요" + ], + "substitute_identifiers": "계정에 로그인할 수 없어요", + "non_latin_words": [ + "계정에", + "로그인할", + "없어요" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "계정에 로그인할 수 없어요", + "python_lower": "계정에 로그인할 수 없어요", + "strip": "계정에 로그인할 수 없어요", + "split_whitespace": [ + "계정에", + "로그인할", + "수", + "없어요" + ], + "split_newline": [ + "계정에 로그인할 수 없어요" + ], + "count_alpha": 11, + "code_point_length": 14, + "english_rescued_by_words": false + } + }, + { + "name": "japanese", + "state": { + "kind": "string", + "value": "アカウントにログインできません" + }, + "state_text": "アカウントにログインできません", + "detect_script": "kana", + "script_profile": { + "kana": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "kana", + "script_profile": { + "kana": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "アカウントにログインできません" + ], + "substitute_identifiers": "アカウントにログインできません", + "non_latin_words": [ + "アカウントにログインできません" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "アカウントにログインできません", + "python_lower": "アカウントにログインできません", + "strip": "アカウントにログインできません", + "split_whitespace": [ + "アカウントにログインできません" + ], + "split_newline": [ + "アカウントにログインできません" + ], + "count_alpha": 15, + "code_point_length": 15, + "english_rescued_by_words": false + } + }, + { + "name": "chinese", + "state": { + "kind": "string", + "value": "我无法登录我的账户" + }, + "state_text": "我无法登录我的账户", + "detect_script": "han", + "script_profile": { + "han": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "han", + "script_profile": { + "han": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "我无法登录我的账户" + ], + "substitute_identifiers": "我无法登录我的账户", + "non_latin_words": [ + "我无法登录我的账户" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "我无法登录我的账户", + "python_lower": "我无法登录我的账户", + "strip": "我无法登录我的账户", + "split_whitespace": [ + "我无法登录我的账户" + ], + "split_newline": [ + "我无法登录我的账户" + ], + "count_alpha": 9, + "code_point_length": 9, + "english_rescued_by_words": false + } + }, + { + "name": "arabic", + "state": { + "kind": "string", + "value": "لا أستطيع تسجيل الدخول" + }, + "state_text": "لا أستطيع تسجيل الدخول", + "detect_script": "arabic", + "script_profile": { + "arabic": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "arabic", + "script_profile": { + "arabic": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "لا", + "أستطيع", + "تسجيل", + "الدخول" + ], + "substitute_identifiers": "لا أستطيع تسجيل الدخول", + "non_latin_words": [ + "لا", + "أستطيع", + "تسجيل", + "الدخول" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "لا أستطيع تسجيل الدخول", + "python_lower": "لا أستطيع تسجيل الدخول", + "strip": "لا أستطيع تسجيل الدخول", + "split_whitespace": [ + "لا", + "أستطيع", + "تسجيل", + "الدخول" + ], + "split_newline": [ + "لا أستطيع تسجيل الدخول" + ], + "count_alpha": 19, + "code_point_length": 22, + "english_rescued_by_words": false + } + }, + { + "name": "hebrew", + "state": { + "kind": "string", + "value": "אני לא יכול להתחבר" + }, + "state_text": "אני לא יכול להתחבר", + "detect_script": "hebrew", + "script_profile": { + "hebrew": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "hebrew", + "script_profile": { + "hebrew": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "אני", + "לא", + "יכול", + "להתחבר" + ], + "substitute_identifiers": "אני לא יכול להתחבר", + "non_latin_words": [ + "אני", + "לא", + "יכול", + "להתחבר" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "אני לא יכול להתחבר", + "python_lower": "אני לא יכול להתחבר", + "strip": "אני לא יכול להתחבר", + "split_whitespace": [ + "אני", + "לא", + "יכול", + "להתחבר" + ], + "split_newline": [ + "אני לא יכול להתחבר" + ], + "count_alpha": 15, + "code_point_length": 18, + "english_rescued_by_words": false + } + }, + { + "name": "greek", + "state": { + "kind": "string", + "value": "Δεν μπορώ να συνδεθώ" + }, + "state_text": "Δεν μπορώ να συνδεθώ", + "detect_script": "greek", + "script_profile": { + "greek": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "greek", + "script_profile": { + "greek": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Δεν", + "μπορώ", + "να", + "συνδεθώ" + ], + "substitute_identifiers": "Δεν μπορώ να συνδεθώ", + "non_latin_words": [ + "μπορώ", + "να", + "συνδεθώ" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "Δεν μπορώ να συνδεθώ", + "python_lower": "δεν μπορώ να συνδεθώ", + "strip": "Δεν μπορώ να συνδεθώ", + "split_whitespace": [ + "Δεν", + "μπορώ", + "να", + "συνδεθώ" + ], + "split_newline": [ + "Δεν μπορώ να συνδεθώ" + ], + "count_alpha": 17, + "code_point_length": 20, + "english_rescued_by_words": false + } + }, + { + "name": "cyrillic", + "state": { + "kind": "string", + "value": "Я не могу войти в свой аккаунт" + }, + "state_text": "Я не могу войти в свой аккаунт", + "detect_script": "cyrillic", + "script_profile": { + "cyrillic": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "cyrillic", + "script_profile": { + "cyrillic": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Я", + "не", + "могу", + "войти", + "в", + "свой", + "аккаунт" + ], + "substitute_identifiers": "Я не могу войти в свой аккаунт", + "non_latin_words": [ + "не", + "могу", + "войти", + "свой", + "аккаунт" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Я не могу войти в свой аккаунт", + "python_lower": "я не могу войти в свой аккаунт", + "strip": "Я не могу войти в свой аккаунт", + "split_whitespace": [ + "Я", + "не", + "могу", + "войти", + "в", + "свой", + "аккаунт" + ], + "split_newline": [ + "Я не могу войти в свой аккаунт" + ], + "count_alpha": 24, + "code_point_length": 30, + "english_rescued_by_words": false + } + }, + { + "name": "thai", + "state": { + "kind": "string", + "value": "ฉันเข้าสู่ระบบไม่ได้" + }, + "state_text": "ฉันเข้าสู่ระบบไม่ได้", + "detect_script": "thai", + "script_profile": { + "thai": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "thai", + "script_profile": { + "thai": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "ฉ", + "นเข", + "าส", + "ระบบไม", + "ได" + ], + "substitute_identifiers": "ฉันเข้าสู่ระบบไม่ได้", + "non_latin_words": [ + "ฉันเขาสระบบไมได" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "ฉันเข้าสู่ระบบไม่ได้", + "python_lower": "ฉันเข้าสู่ระบบไม่ได้", + "strip": "ฉันเข้าสู่ระบบไม่ได้", + "split_whitespace": [ + "ฉันเข้าสู่ระบบไม่ได้" + ], + "split_newline": [ + "ฉันเข้าสู่ระบบไม่ได้" + ], + "count_alpha": 14, + "code_point_length": 20, + "english_rescued_by_words": false + } + }, + { + "name": "tamil", + "state": { + "kind": "string", + "value": "என் கணக்கில் உள்நுழைண முடியவில்லை" + }, + "state_text": "என் கணக்கில் உள்நுழைண முடியவில்லை", + "detect_script": "tamil", + "script_profile": { + "tamil": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "tamil", + "script_profile": { + "tamil": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "என", + "கணக", + "க", + "ல", + "உள", + "ந", + "ழ", + "ண", + "ம", + "ட", + "யவ", + "ல", + "ல" + ], + "substitute_identifiers": "என் கணக்கில் உள்நுழைண முடியவில்லை", + "non_latin_words": [ + "என", + "கணககில", + "உளநுழைண", + "முடியவிலலை" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "என் கணக்கில் உள்நுழைண முடியவில்லை", + "python_lower": "என் கணக்கில் உள்நுழைண முடியவில்லை", + "strip": "என் கணக்கில் உள்நுழைண முடியவில்லை", + "split_whitespace": [ + "என்", + "கணக்கில்", + "உள்நுழைண", + "முடியவில்லை" + ], + "split_newline": [ + "என் கணக்கில் உள்நுழைண முடியவில்லை" + ], + "count_alpha": 18, + "code_point_length": 33, + "english_rescued_by_words": false + } + }, + { + "name": "amharic", + "state": { + "kind": "string", + "value": "አሰላም ከመገባት አልችልም" + }, + "state_text": "አሰላም ከመገባት አልችልም", + "detect_script": "ethiopic", + "script_profile": { + "ethiopic": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "ethiopic", + "script_profile": { + "ethiopic": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "አሰላም", + "ከመገባት", + "አልችልም" + ], + "substitute_identifiers": "አሰላም ከመገባት አልችልም", + "non_latin_words": [ + "አሰላም", + "ከመገባት", + "አልችልም" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false + ], + "blank_upper_runs": "አሰላም ከመገባት አልችልም", + "python_lower": "አሰላም ከመገባት አልችልም", + "strip": "አሰላም ከመገባት አልችልም", + "split_whitespace": [ + "አሰላም", + "ከመገባት", + "አልችልም" + ], + "split_newline": [ + "አሰላም ከመገባት አልችልም" + ], + "count_alpha": 14, + "code_point_length": 16, + "english_rescued_by_words": false + } + }, + { + "name": "georgian", + "state": { + "kind": "string", + "value": "ვერ ვახერხე ანგარიში" + }, + "state_text": "ვერ ვახერხე ანგარიში", + "detect_script": "georgian", + "script_profile": { + "georgian": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "georgian", + "script_profile": { + "georgian": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "ვერ", + "ვახერხე", + "ანგარიში" + ], + "substitute_identifiers": "ვერ ვახერხე ანგარიში", + "non_latin_words": [ + "ვერ", + "ვახერხე", + "ანგარიში" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false + ], + "blank_upper_runs": "ვერ ვახერხე ანგარიში", + "python_lower": "ვერ ვახერხე ანგარიში", + "strip": "ვერ ვახერხე ანგარიში", + "split_whitespace": [ + "ვერ", + "ვახერხე", + "ანგარიში" + ], + "split_newline": [ + "ვერ ვახერხე ანგარიში" + ], + "count_alpha": 18, + "code_point_length": 20, + "english_rescued_by_words": false + } + }, + { + "name": "armenian", + "state": { + "kind": "string", + "value": "Ես չեմ կարողանում մտնել" + }, + "state_text": "Ես չեմ կարողանում մտնել", + "detect_script": "armenian", + "script_profile": { + "armenian": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "armenian", + "script_profile": { + "armenian": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Ես", + "չեմ", + "կարողանում", + "մտնել" + ], + "substitute_identifiers": "Ես չեմ կարողանում մտնել", + "non_latin_words": [ + "չեմ", + "կարողանում", + "մտնել" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "Ես չեմ կարողանում մտնել", + "python_lower": "ես չեմ կարողանում մտնել", + "strip": "Ես չեմ կարողանում մտնել", + "split_whitespace": [ + "Ես", + "չեմ", + "կարողանում", + "մտնել" + ], + "split_newline": [ + "Ես չեմ կարողանում մտնել" + ], + "count_alpha": 20, + "code_point_length": 23, + "english_rescued_by_words": false + } + }, + { + "name": "unlisted-script-kawi", + "state": { + "kind": "string", + "value": "𑼀𑼁𑼂𑼃𑼄" + }, + "state_text": "𑼀𑼁𑼂𑼃𑼄", + "detect_script": "other", + "script_profile": { + "other": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "other", + "script_profile": { + "other": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "𑼂", + "𑼄" + ], + "substitute_identifiers": "𑼀𑼁𑼂𑼃𑼄", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "𑼀𑼁𑼂𑼃𑼄", + "python_lower": "𑼀𑼁𑼂𑼃𑼄", + "strip": "𑼀𑼁𑼂𑼃𑼄", + "split_whitespace": [ + "𑼀𑼁𑼂𑼃𑼄" + ], + "split_newline": [ + "𑼀𑼁𑼂𑼃𑼄" + ], + "count_alpha": 2, + "code_point_length": 5, + "english_rescued_by_words": false + } + }, + { + "name": "cjk-extension-b", + "state": { + "kind": "string", + "value": "𠀀𠀁𠀂" + }, + "state_text": "𠀀𠀁𠀂", + "detect_script": "other", + "script_profile": { + "other": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "other", + "script_profile": { + "other": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "𠀀𠀁𠀂" + ], + "substitute_identifiers": "𠀀𠀁𠀂", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "𠀀𠀁𠀂", + "python_lower": "𠀀𠀁𠀂", + "strip": "𠀀𠀁𠀂", + "split_whitespace": [ + "𠀀𠀁𠀂" + ], + "split_newline": [ + "𠀀𠀁𠀂" + ], + "count_alpha": 3, + "code_point_length": 3, + "english_rescued_by_words": false + } + }, + { + "name": "fullwidth-latin", + "state": { + "kind": "string", + "value": "Hello World" + }, + "state_text": "Hello World", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "Hello", + "World" + ], + "substitute_identifiers": "Hello World", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false + ], + "blank_upper_runs": "Hello World", + "python_lower": "hello world", + "strip": "Hello World", + "split_whitespace": [ + "Hello", + "World" + ], + "split_newline": [ + "Hello World" + ], + "count_alpha": 10, + "code_point_length": 11, + "english_rescued_by_words": false + } + }, + { + "name": "latin-ext-additional", + "state": { + "kind": "string", + "value": "ẞẽễḍḥ" + }, + "state_text": "ẞẽễḍḥ", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.2, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.2, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "ẞẽễḍḥ" + ], + "substitute_identifiers": "ẞẽễḍḥ", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false + ], + "blank_upper_runs": "ẞẽễḍḥ", + "python_lower": "ßẽễḍḥ", + "strip": "ẞẽễḍḥ", + "split_whitespace": [ + "ẞẽễḍḥ" + ], + "split_newline": [ + "ẞẽễḍḥ" + ], + "count_alpha": 5, + "code_point_length": 5, + "english_rescued_by_words": false + } + }, + { + "name": "ipa-pronunciation", + "state": { + "kind": "string", + "value": "The name is pronounced [vlɐˈdʲimʲɨr] in Russian and that is all" + }, + "state_text": "The name is pronounced [vlɐˈdʲimʲɨr] in Russian and that is all", + "detect_script": "latin", + "script_profile": { + "latin": 0.9411764705882353, + "other": 0.058823529411764705 + }, + "latin_profile": { + "language": "en", + "english_hits": 6, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 0.9411764705882353, + "other": 0.058823529411764705 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0588, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "The", + "name", + "is", + "pronounced", + "vlɐˈdʲimʲɨr", + "in", + "Russian", + "and", + "that", + "is", + "all" + ], + "substitute_identifiers": "The name is pronounced [vlɐˈdʲimʲɨr] in Russian and that is all", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": true, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The name is pronounced [vlɐˈdʲimʲɨr] in Russian and that is all", + "python_lower": "the name is pronounced [vlɐˈdʲimʲɨr] in russian and that is all", + "strip": "The name is pronounced [vlɐˈdʲimʲɨr] in Russian and that is all", + "split_whitespace": [ + "The", + "name", + "is", + "pronounced", + "[vlɐˈdʲimʲɨr]", + "in", + "Russian", + "and", + "that", + "is", + "all" + ], + "split_newline": [ + "The name is pronounced [vlɐˈdʲimʲɨr] in Russian and that is all" + ], + "count_alpha": 51, + "code_point_length": 63, + "english_rescued_by_words": true + } + }, + { + "name": "greek-symbol-in-english", + "state": { + "kind": "string", + "value": "Set α to 0.05 and then re-run the whole evaluation again" + }, + "state_text": "Set α to 0.05 and then re-run the whole evaluation again", + "detect_script": "latin", + "script_profile": { + "latin": 0.975609756097561, + "greek": 0.024390243902439025 + }, + "latin_profile": { + "language": "en", + "english_hits": 3, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 0.975609756097561, + "greek": 0.024390243902439025 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0244, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "Set", + "α", + "to", + "and", + "then", + "re", + "run", + "the", + "whole", + "evaluation", + "again" + ], + "substitute_identifiers": "Set α to and then re-run the whole evaluation again", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Set α to 0.05 and then re-run the whole evaluation again", + "python_lower": "set α to 0.05 and then re-run the whole evaluation again", + "strip": "Set α to 0.05 and then re-run the whole evaluation again", + "split_whitespace": [ + "Set", + "α", + "to", + "0.05", + "and", + "then", + "re-run", + "the", + "whole", + "evaluation", + "again" + ], + "split_newline": [ + "Set α to 0.05 and then re-run the whole evaluation again" + ], + "count_alpha": 41, + "code_point_length": 56, + "english_rescued_by_words": true + } + }, + { + "name": "cyrillic-proper-name", + "state": { + "kind": "string", + "value": "The reviewer was Дмитрий Петрович and he approved the change" + }, + "state_text": "The reviewer was Дмитрий Петрович and he approved the change", + "detect_script": "latin", + "script_profile": { + "latin": 0.7058823529411765, + "cyrillic": 0.29411764705882354 + }, + "latin_profile": { + "language": "en", + "english_hits": 4, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 0.7058823529411765, + "cyrillic": 0.29411764705882354 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.2941, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "The", + "reviewer", + "was", + "Дмитрий", + "Петрович", + "and", + "he", + "approved", + "the", + "change" + ], + "substitute_identifiers": "The reviewer was Дмитрий Петрович and he approved the change", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The reviewer was Дмитрий Петрович and he approved the change", + "python_lower": "the reviewer was дмитрий петрович and he approved the change", + "strip": "The reviewer was Дмитрий Петрович and he approved the change", + "split_whitespace": [ + "The", + "reviewer", + "was", + "Дмитрий", + "Петрович", + "and", + "he", + "approved", + "the", + "change" + ], + "split_newline": [ + "The reviewer was Дмитрий Петрович and he approved the change" + ], + "count_alpha": 51, + "code_point_length": 60, + "english_rescued_by_words": true + } + }, + { + "name": "cyrillic-name-with-combining", + "state": { + "kind": "string", + "value": "The reviewer was Влади́мир and he approved the change" + }, + "state_text": "The reviewer was Влади́мир and he approved the change", + "detect_script": "latin", + "script_profile": { + "latin": 0.8181818181818182, + "cyrillic": 0.18181818181818182 + }, + "latin_profile": { + "language": "en", + "english_hits": 4, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 0.8181818181818182, + "cyrillic": 0.18181818181818182 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.1818, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "The", + "reviewer", + "was", + "Влади", + "мир", + "and", + "he", + "approved", + "the", + "change" + ], + "substitute_identifiers": "The reviewer was Влади́мир and he approved the change", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The reviewer was Влади́мир and he approved the change", + "python_lower": "the reviewer was влади́мир and he approved the change", + "strip": "The reviewer was Влади́мир and he approved the change", + "split_whitespace": [ + "The", + "reviewer", + "was", + "Влади́мир", + "and", + "he", + "approved", + "the", + "change" + ], + "split_newline": [ + "The reviewer was Влади́мир and he approved the change" + ], + "count_alpha": 44, + "code_point_length": 53, + "english_rescued_by_words": true + } + }, + { + "name": "latin-brand-in-cjk", + "state": { + "kind": "string", + "value": "ACME-ORDER-99281-XYZ 注文が届きません" + }, + "state_text": "ACME-ORDER-99281-XYZ 注文が届きません", + "detect_script": "latin", + "script_profile": { + "latin": 0.6, + "han": 0.15, + "kana": 0.25 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "kana", + "script_profile": { + "latin": 0.6, + "han": 0.15, + "kana": 0.25 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.4, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "ACME", + "ORDER", + "XYZ", + "注文が届きません" + ], + "substitute_identifiers": "ACME-ORDER-99281-XYZ 注文が届きません", + "non_latin_words": [ + "注文", + "きません" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false + ], + "blank_upper_runs": " - -99281- 注文が届きません", + "python_lower": "acme-order-99281-xyz 注文が届きません", + "strip": "ACME-ORDER-99281-XYZ 注文が届きません", + "split_whitespace": [ + "ACME-ORDER-99281-XYZ", + "注文が届きません" + ], + "split_newline": [ + "ACME-ORDER-99281-XYZ 注文が届きません" + ], + "count_alpha": 20, + "code_point_length": 29, + "english_rescued_by_words": false + } + }, + { + "name": "latin-plurality-over-cjk", + "state": { + "kind": "string", + "value": "Order ACME-99281 shipped but the label is wrong 注文番号が違いますよ" + }, + "state_text": "Order ACME-99281 shipped but the label is wrong 注文番号が違いますよ", + "detect_script": "latin", + "script_profile": { + "latin": 0.7727272727272727, + "han": 0.11363636363636363, + "kana": 0.11363636363636363 + }, + "latin_profile": { + "language": "en", + "english_hits": 3, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "han", + "script_profile": { + "latin": 0.7727272727272727, + "han": 0.11363636363636363, + "kana": 0.11363636363636363 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.2273, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Order", + "ACME", + "shipped", + "but", + "the", + "label", + "is", + "wrong", + "注文番号が違いますよ" + ], + "substitute_identifiers": "Order ACME-99281 shipped but the label is wrong 注文番号が違いますよ", + "non_latin_words": [ + "注文番号", + "いますよ" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Order -99281 shipped but the label is wrong 注文番号が違いますよ", + "python_lower": "order acme-99281 shipped but the label is wrong 注文番号が違いますよ", + "strip": "Order ACME-99281 shipped but the label is wrong 注文番号が違いますよ", + "split_whitespace": [ + "Order", + "ACME-99281", + "shipped", + "but", + "the", + "label", + "is", + "wrong", + "注文番号が違いますよ" + ], + "split_newline": [ + "Order ACME-99281 shipped but the label is wrong 注文番号が違いますよ" + ], + "count_alpha": 44, + "code_point_length": 58, + "english_rescued_by_words": true + } + }, + { + "name": "urls-and-emails", + "state": { + "kind": "string", + "value": "See github.com/acme/repo and mail user@acme.com about v1.2.3 in the U.S.A." + }, + "state_text": "See github.com/acme/repo and mail user@acme.com about v1.2.3 in the U.S.A.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 3, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "See", + "github", + "com", + "acme", + "repo", + "and", + "mail", + "user", + "acme", + "com", + "about", + "v", + "in", + "the", + "U", + "S", + "A" + ], + "substitute_identifiers": "See /acme/repo and mail about in the .", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + true, + false, + false, + true, + false, + true, + false, + false, + true + ], + "blank_upper_runs": "See github.com/acme/repo and mail user@acme.com about v1.2.3 in the U.S.A.", + "python_lower": "see github.com/acme/repo and mail user@acme.com about v1.2.3 in the u.s.a.", + "strip": "See github.com/acme/repo and mail user@acme.com about v1.2.3 in the U.S.A.", + "split_whitespace": [ + "See", + "github.com/acme/repo", + "and", + "mail", + "user@acme.com", + "about", + "v1.2.3", + "in", + "the", + "U.S.A." + ], + "split_newline": [ + "See github.com/acme/repo and mail user@acme.com about v1.2.3 in the U.S.A." + ], + "count_alpha": 52, + "code_point_length": 74, + "english_rescued_by_words": true + } + }, + { + "name": "url-heavy-portuguese-looking", + "state": { + "kind": "string", + "value": "github.com e os.path com o.com de.com na.com" + }, + "state_text": "github.com e os.path com o.com de.com na.com", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "github", + "com", + "e", + "os", + "path", + "com", + "o", + "com", + "de", + "com", + "na", + "com" + ], + "substitute_identifiers": " e com ", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + true, + false, + true, + false, + true, + true, + true + ], + "blank_upper_runs": "github.com e os.path com o.com de.com na.com", + "python_lower": "github.com e os.path com o.com de.com na.com", + "strip": "github.com e os.path com o.com de.com na.com", + "split_whitespace": [ + "github.com", + "e", + "os.path", + "com", + "o.com", + "de.com", + "na.com" + ], + "split_newline": [ + "github.com e os.path com o.com de.com na.com" + ], + "count_alpha": 33, + "code_point_length": 44, + "english_rescued_by_words": false + } + }, + { + "name": "leading-dot-identifier", + "state": { + "kind": "string", + "value": ".com .org .net and the rest of the list is here now" + }, + "state_text": ".com .org .net and the rest of the list is here now", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 5, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "com", + "org", + "net", + "and", + "the", + "rest", + "of", + "the", + "list", + "is", + "here", + "now" + ], + "substitute_identifiers": " and the rest of the list is here now", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": ".com .org .net and the rest of the list is here now", + "python_lower": ".com .org .net and the rest of the list is here now", + "strip": ".com .org .net and the rest of the list is here now", + "split_whitespace": [ + ".com", + ".org", + ".net", + "and", + "the", + "rest", + "of", + "the", + "list", + "is", + "here", + "now" + ], + "split_newline": [ + ".com .org .net and the rest of the list is here now" + ], + "count_alpha": 37, + "code_point_length": 51, + "english_rescued_by_words": true + } + }, + { + "name": "at-after-dot", + "state": { + "kind": "string", + "value": "a.@b and the rest of this sentence is plain English prose here" + }, + "state_text": "a.@b and the rest of this sentence is plain English prose here", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 5, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "a", + "b", + "and", + "the", + "rest", + "of", + "this", + "sentence", + "is", + "plain", + "English", + "prose", + "here" + ], + "substitute_identifiers": "a. and the rest of this sentence is plain English prose here", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "a.@b and the rest of this sentence is plain English prose here", + "python_lower": "a.@b and the rest of this sentence is plain english prose here", + "strip": "a.@b and the rest of this sentence is plain English prose here", + "split_whitespace": [ + "a.@b", + "and", + "the", + "rest", + "of", + "this", + "sentence", + "is", + "plain", + "English", + "prose", + "here" + ], + "split_newline": [ + "a.@b and the rest of this sentence is plain English prose here" + ], + "count_alpha": 49, + "code_point_length": 62, + "english_rescued_by_words": true + } + }, + { + "name": "sentence-final-period", + "state": { + "kind": "string", + "value": "Il pacco e arrivato. Non trovo la fattura nel portale adesso." + }, + "state_text": "Il pacco e arrivato. Non trovo la fattura nel portale adesso.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "it", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "it", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Il", + "pacco", + "e", + "arrivato", + "Non", + "trovo", + "la", + "fattura", + "nel", + "portale", + "adesso" + ], + "substitute_identifiers": "Il pacco e arrivato. Non trovo la fattura nel portale adesso.", + "non_latin_words": [], + "named_prose_language": "it", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Il pacco e arrivato. Non trovo la fattura nel portale adesso.", + "python_lower": "il pacco e arrivato. non trovo la fattura nel portale adesso.", + "strip": "Il pacco e arrivato. Non trovo la fattura nel portale adesso.", + "split_whitespace": [ + "Il", + "pacco", + "e", + "arrivato.", + "Non", + "trovo", + "la", + "fattura", + "nel", + "portale", + "adesso." + ], + "split_newline": [ + "Il pacco e arrivato. Non trovo la fattura nel portale adesso." + ], + "count_alpha": 49, + "code_point_length": 61, + "english_rescued_by_words": false + } + }, + { + "name": "code-line", + "state": { + "kind": "string", + "value": "result = round(el, 2); os.path.join(a, b)" + }, + "state_text": "result = round(el, 2); os.path.join(a, b)", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "result", + "round", + "el", + "os", + "path", + "join", + "a", + "b" + ], + "substitute_identifiers": "result = round(el, 2); (a, b)", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": true, + "has_joined_tokens": [ + false, + false, + false, + false, + true, + false + ], + "blank_upper_runs": "result = round(el, 2); os.path.join(a, b)", + "python_lower": "result = round(el, 2); os.path.join(a, b)", + "strip": "result = round(el, 2); os.path.join(a, b)", + "split_whitespace": [ + "result", + "=", + "round(el,", + "2);", + "os.path.join(a,", + "b)" + ], + "split_newline": [ + "result = round(el, 2); os.path.join(a, b)" + ], + "count_alpha": 25, + "code_point_length": 41, + "english_rescued_by_words": false + } + }, + { + "name": "acronym-heavy-english", + "state": { + "kind": "string", + "value": "The MON LA EST COM DES game was moved to the next week entirely" + }, + "state_text": "The MON LA EST COM DES game was moved to the next week entirely", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 4, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "The", + "MON", + "LA", + "EST", + "COM", + "DES", + "game", + "was", + "moved", + "to", + "the", + "next", + "week", + "entirely" + ], + "substitute_identifiers": "The MON LA EST COM DES game was moved to the next week entirely", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The game was moved to the next week entirely", + "python_lower": "the mon la est com des game was moved to the next week entirely", + "strip": "The MON LA EST COM DES game was moved to the next week entirely", + "split_whitespace": [ + "The", + "MON", + "LA", + "EST", + "COM", + "DES", + "game", + "was", + "moved", + "to", + "the", + "next", + "week", + "entirely" + ], + "split_newline": [ + "The MON LA EST COM DES game was moved to the next week entirely" + ], + "count_alpha": 50, + "code_point_length": 63, + "english_rescued_by_words": true + } + }, + { + "name": "all-caps-portuguese", + "state": { + "kind": "string", + "value": "NÃO CONSIGO ENTRAR NA MINHA CONTA E A SENHA NÃO FUNCIONA" + }, + "state_text": "NÃO CONSIGO ENTRAR NA MINHA CONTA E A SENHA NÃO FUNCIONA", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "pt", + "english_hits": 0, + "diacritic_rate": 0.03571428571428571, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0357, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "NÃO", + "CONSIGO", + "ENTRAR", + "NA", + "MINHA", + "CONTA", + "E", + "A", + "SENHA", + "NÃO", + "FUNCIONA" + ], + "substitute_identifiers": "NÃO CONSIGO ENTRAR NA MINHA CONTA E A SENHA NÃO FUNCIONA", + "non_latin_words": [], + "named_prose_language": "pt", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": " E A ", + "python_lower": "não consigo entrar na minha conta e a senha não funciona", + "strip": "NÃO CONSIGO ENTRAR NA MINHA CONTA E A SENHA NÃO FUNCIONA", + "split_whitespace": [ + "NÃO", + "CONSIGO", + "ENTRAR", + "NA", + "MINHA", + "CONTA", + "E", + "A", + "SENHA", + "NÃO", + "FUNCIONA" + ], + "split_newline": [ + "NÃO CONSIGO ENTRAR NA MINHA CONTA E A SENHA NÃO FUNCIONA" + ], + "count_alpha": 46, + "code_point_length": 56, + "english_rescued_by_words": false + } + }, + { + "name": "joined-compound-names", + "state": { + "kind": "string", + "value": "Nav/Com and OS/2 and C:\\DOS\\mode were all on the same list" + }, + "state_text": "Nav/Com and OS/2 and C:\\DOS\\mode were all on the same list", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 5, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "Nav", + "Com", + "and", + "OS", + "and", + "C", + "DOS", + "mode", + "were", + "all", + "on", + "the", + "same", + "list" + ], + "substitute_identifiers": "Nav/Com and OS/2 and C:\\DOS\\mode were all on the same list", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + true, + false, + true, + false, + true, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Nav/Com and /2 and C:\\ \\mode were all on the same list", + "python_lower": "nav/com and os/2 and c:\\dos\\mode were all on the same list", + "strip": "Nav/Com and OS/2 and C:\\DOS\\mode were all on the same list", + "split_whitespace": [ + "Nav/Com", + "and", + "OS/2", + "and", + "C:\\DOS\\mode", + "were", + "all", + "on", + "the", + "same", + "list" + ], + "split_newline": [ + "Nav/Com and OS/2 and C:\\DOS\\mode were all on the same list" + ], + "count_alpha": 42, + "code_point_length": 58, + "english_rescued_by_words": true + } + }, + { + "name": "nbsp-joined-words", + "state": { + "kind": "string", + "value": "Je ne peux pas me connecter a mon compte du tout" + }, + "state_text": "Je ne peux pas me connecter a mon compte du tout", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "fr", + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "fr", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Je", + "ne", + "peux", + "pas", + "me", + "connecter", + "a", + "mon", + "compte", + "du", + "tout" + ], + "substitute_identifiers": "Je ne peux pas me connecter a mon compte du tout", + "non_latin_words": [], + "named_prose_language": "fr", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Je ne peux pas me connecter a mon compte du tout", + "python_lower": "je ne peux pas me connecter a mon compte du tout", + "strip": "Je ne peux pas me connecter a mon compte du tout", + "split_whitespace": [ + "Je", + "ne", + "peux", + "pas", + "me", + "connecter", + "a", + "mon", + "compte", + "du", + "tout" + ], + "split_newline": [ + "Je ne peux pas me connecter a mon compte du tout" + ], + "count_alpha": 38, + "code_point_length": 48, + "english_rescued_by_words": false + } + }, + { + "name": "final-sigma", + "state": { + "kind": "string", + "value": "ΔΕΝ ΜΠΟΡΩ ΝΑ ΣΥΝΔΕΘΩ" + }, + "state_text": "ΔΕΝ ΜΠΟΡΩ ΝΑ ΣΥΝΔΕΘΩ", + "detect_script": "greek", + "script_profile": { + "greek": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "greek", + "script_profile": { + "greek": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 1.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "ΔΕΝ", + "ΜΠΟΡΩ", + "ΝΑ", + "ΣΥΝΔΕΘΩ" + ], + "substitute_identifiers": "ΔΕΝ ΜΠΟΡΩ ΝΑ ΣΥΝΔΕΘΩ", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": " ", + "python_lower": "δεν μπορω να συνδεθω", + "strip": "ΔΕΝ ΜΠΟΡΩ ΝΑ ΣΥΝΔΕΘΩ", + "split_whitespace": [ + "ΔΕΝ", + "ΜΠΟΡΩ", + "ΝΑ", + "ΣΥΝΔΕΘΩ" + ], + "split_newline": [ + "ΔΕΝ ΜΠΟΡΩ ΝΑ ΣΥΝΔΕΘΩ" + ], + "count_alpha": 17, + "code_point_length": 20, + "english_rescued_by_words": false + } + }, + { + "name": "english-with-portuguese-line", + "state": { + "kind": "string", + "value": "Customer ticket #4471\nNao consigo entrar na minha conta e a senha nao funciona\nAgent: asked for a screenshot" + }, + "state_text": "Customer ticket #4471\nNao consigo entrar na minha conta e a senha nao funciona\nAgent: asked for a screenshot", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "pt", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Customer", + "ticket", + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "Agent", + "asked", + "for", + "a", + "screenshot" + ], + "substitute_identifiers": "Customer ticket #4471\nNao consigo entrar na minha conta e a senha nao funciona\nAgent: asked for a screenshot", + "non_latin_words": [], + "named_prose_language": "pt", + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Customer ticket #4471\nNao consigo entrar na minha conta e a senha nao funciona\nAgent: asked for a screenshot", + "python_lower": "customer ticket #4471\nnao consigo entrar na minha conta e a senha nao funciona\nagent: asked for a screenshot", + "strip": "Customer ticket #4471\nNao consigo entrar na minha conta e a senha nao funciona\nAgent: asked for a screenshot", + "split_whitespace": [ + "Customer", + "ticket", + "#4471", + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "Agent:", + "asked", + "for", + "a", + "screenshot" + ], + "split_newline": [ + "Customer ticket #4471", + "Nao consigo entrar na minha conta e a senha nao funciona", + "Agent: asked for a screenshot" + ], + "count_alpha": 84, + "code_point_length": 108, + "english_rescued_by_words": false + } + }, + { + "name": "english-stack-trace-with-german-line", + "state": { + "kind": "string", + "value": "Traceback (most recent call last):\n File \"app.py\", line 12, in handler\n raise ValueError(x)\nIch kann mich nicht in mein Konto einloggen und das Passwort geht nicht" + }, + "state_text": "Traceback (most recent call last):\n File \"app.py\", line 12, in handler\n raise ValueError(x)\nIch kann mich nicht in mein Konto einloggen und das Passwort geht nicht", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "de", + "english_hits": 2, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "de", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "Traceback", + "most", + "recent", + "call", + "last", + "File", + "app", + "py", + "line", + "in", + "handler", + "raise", + "ValueError", + "x", + "Ich", + "kann", + "mich", + "nicht", + "in", + "mein", + "Konto", + "einloggen", + "und", + "das", + "Passwort", + "geht", + "nicht" + ], + "substitute_identifiers": "Traceback (most recent call last):\n File \" \", line 12, in handler\n raise ValueError(x)\nIch kann mich nicht in mein Konto einloggen und das Passwort geht nicht", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": true, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Traceback (most recent call last):\n File \"app.py\", line 12, in handler\n raise ValueError(x)\nIch kann mich nicht in mein Konto einloggen und das Passwort geht nicht", + "python_lower": "traceback (most recent call last):\n file \"app.py\", line 12, in handler\n raise valueerror(x)\nich kann mich nicht in mein konto einloggen und das passwort geht nicht", + "strip": "Traceback (most recent call last):\n File \"app.py\", line 12, in handler\n raise ValueError(x)\nIch kann mich nicht in mein Konto einloggen und das Passwort geht nicht", + "split_whitespace": [ + "Traceback", + "(most", + "recent", + "call", + "last):", + "File", + "\"app.py\",", + "line", + "12,", + "in", + "handler", + "raise", + "ValueError(x)", + "Ich", + "kann", + "mich", + "nicht", + "in", + "mein", + "Konto", + "einloggen", + "und", + "das", + "Passwort", + "geht", + "nicht" + ], + "split_newline": [ + "Traceback (most recent call last):", + " File \"app.py\", line 12, in handler", + " raise ValueError(x)", + "Ich kann mich nicht in mein Konto einloggen und das Passwort geht nicht" + ], + "count_alpha": 124, + "code_point_length": 167, + "english_rescued_by_words": false + } + }, + { + "name": "single-line-english-only", + "state": { + "kind": "string", + "value": "I cannot log in to my account at all today" + }, + "state_text": "I cannot log in to my account at all today", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 4, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "I", + "cannot", + "log", + "in", + "to", + "my", + "account", + "at", + "all", + "today" + ], + "substitute_identifiers": "I cannot log in to my account at all today", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "I cannot log in to my account at all today", + "python_lower": "i cannot log in to my account at all today", + "strip": "I cannot log in to my account at all today", + "split_whitespace": [ + "I", + "cannot", + "log", + "in", + "to", + "my", + "account", + "at", + "all", + "today" + ], + "split_newline": [ + "I cannot log in to my account at all today" + ], + "count_alpha": 33, + "code_point_length": 42, + "english_rescued_by_words": true + } + }, + { + "name": "short-lines-under-seven", + "state": { + "kind": "string", + "value": "ok\nno\nyes\nfail\nabcdef" + }, + "state_text": "ok\nno\nyes\nfail\nabcdef", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "ok", + "no", + "yes", + "fail", + "abcdef" + ], + "substitute_identifiers": "ok\nno\nyes\nfail\nabcdef", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false + ], + "blank_upper_runs": "ok\nno\nyes\nfail\nabcdef", + "python_lower": "ok\nno\nyes\nfail\nabcdef", + "strip": "ok\nno\nyes\nfail\nabcdef", + "split_whitespace": [ + "ok", + "no", + "yes", + "fail", + "abcdef" + ], + "split_newline": [ + "ok", + "no", + "yes", + "fail", + "abcdef" + ], + "count_alpha": 17, + "code_point_length": 21, + "english_rescued_by_words": false + } + }, + { + "name": "one-greek-letter-in-thirty-two", + "state": { + "kind": "string", + "value": "Set the threshold values to Α and rerun" + }, + "state_text": "Set the threshold values to Α and rerun", + "detect_script": "latin", + "script_profile": { + "latin": 0.96875, + "greek": 0.03125 + }, + "latin_profile": { + "language": "en", + "english_hits": 3, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 0.96875, + "greek": 0.03125 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0312, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "Set", + "the", + "threshold", + "values", + "to", + "Α", + "and", + "rerun" + ], + "substitute_identifiers": "Set the threshold values to Α and rerun", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Set the threshold values to Α and rerun", + "python_lower": "set the threshold values to α and rerun", + "strip": "Set the threshold values to Α and rerun", + "split_whitespace": [ + "Set", + "the", + "threshold", + "values", + "to", + "Α", + "and", + "rerun" + ], + "split_newline": [ + "Set the threshold values to Α and rerun" + ], + "count_alpha": 32, + "code_point_length": 39, + "english_rescued_by_words": true + } + }, + { + "name": "one-diacritic-in-thirty-two", + "state": { + "kind": "string", + "value": "café xxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + "state_text": "café xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.03125, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0312, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "café", + "xxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "substitute_identifiers": "café xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false + ], + "blank_upper_runs": "café xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "python_lower": "café xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "strip": "café xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "split_whitespace": [ + "café", + "xxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "split_newline": [ + "café xxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "count_alpha": 31, + "code_point_length": 32, + "english_rescued_by_words": false + } + }, + { + "name": "three-non-latin-scripts", + "state": { + "kind": "string", + "value": "ΑΒ БВ 漢字 and some latin words here" + }, + "state_text": "ΑΒ БВ 漢字 and some latin words here", + "detect_script": "latin", + "script_profile": { + "latin": 0.7777777777777778, + "greek": 0.07407407407407407, + "cyrillic": 0.07407407407407407, + "han": 0.07407407407407407 + }, + "latin_profile": { + "language": "en", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "greek", + "script_profile": { + "latin": 0.7777777777777778, + "greek": 0.07407407407407407, + "cyrillic": 0.07407407407407407, + "han": 0.07407407407407407 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.2222, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "ΑΒ", + "БВ", + "漢字", + "and", + "some", + "latin", + "words", + "here" + ], + "substitute_identifiers": "ΑΒ БВ 漢字 and some latin words here", + "non_latin_words": [ + "漢字" + ], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": " 漢字 and some latin words here", + "python_lower": "αβ бв 漢字 and some latin words here", + "strip": "ΑΒ БВ 漢字 and some latin words here", + "split_whitespace": [ + "ΑΒ", + "БВ", + "漢字", + "and", + "some", + "latin", + "words", + "here" + ], + "split_newline": [ + "ΑΒ БВ 漢字 and some latin words here" + ], + "count_alpha": 27, + "code_point_length": 34, + "english_rescued_by_words": false + } + }, + { + "name": "latin-and-greek-tied", + "state": { + "kind": "string", + "value": "abc ΑΒΓ" + }, + "state_text": "abc ΑΒΓ", + "detect_script": "greek", + "script_profile": { + "latin": 0.5, + "greek": 0.5 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "greek", + "script_profile": { + "latin": 0.5, + "greek": 0.5 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.5, + "mixed_segment": null + }, + "is_english": false, + "helpers": { + "words": [ + "abc", + "ΑΒΓ" + ], + "substitute_identifiers": "abc ΑΒΓ", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false + ], + "blank_upper_runs": "abc ", + "python_lower": "abc αβγ", + "strip": "abc ΑΒΓ", + "split_whitespace": [ + "abc", + "ΑΒΓ" + ], + "split_newline": [ + "abc ΑΒΓ" + ], + "count_alpha": 6, + "code_point_length": 7, + "english_rescued_by_words": false + } + }, + { + "name": "collision-word-twice", + "state": { + "kind": "string", + "value": "todo todo bueno bueno" + }, + "state_text": "todo todo bueno bueno", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "todo", + "todo", + "bueno", + "bueno" + ], + "substitute_identifiers": "todo todo bueno bueno", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "todo todo bueno bueno", + "python_lower": "todo todo bueno bueno", + "strip": "todo todo bueno bueno", + "split_whitespace": [ + "todo", + "todo", + "bueno", + "bueno" + ], + "split_newline": [ + "todo todo bueno bueno" + ], + "count_alpha": 18, + "code_point_length": 21, + "english_rescued_by_words": false + } + }, + { + "name": "margin-exactly-one-over-english", + "state": { + "kind": "string", + "value": "der nicht the house" + }, + "state_text": "der nicht the house", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "der", + "nicht", + "the", + "house" + ], + "substitute_identifiers": "der nicht the house", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false + ], + "blank_upper_runs": "der nicht the house", + "python_lower": "der nicht the house", + "strip": "der nicht the house", + "split_whitespace": [ + "der", + "nicht", + "the", + "house" + ], + "split_newline": [ + "der nicht the house" + ], + "count_alpha": 16, + "code_point_length": 19, + "english_rescued_by_words": false + } + }, + { + "name": "english-note-with-portuguese-line", + "state": { + "kind": "string", + "value": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today." + }, + "state_text": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 18, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum" + }, + "is_english": false, + "helpers": { + "words": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot", + "The", + "browser", + "console", + "shows", + "no", + "errors", + "and", + "the", + "network", + "tab", + "looks", + "clean", + "to", + "me", + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "de", + "jeito", + "nenhum", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today" + ], + "substitute_identifiers": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "python_lower": "the customer opened this ticket yesterday and we have asked for a screenshot.\nthe browser console shows no errors and the network tab looks clean to me.\nnao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nwe will escalate this to the platform team if it is not resolved today.", + "strip": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "split_whitespace": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot.", + "The", + "browser", + "console", + "shows", + "no", + "errors", + "and", + "the", + "network", + "tab", + "looks", + "clean", + "to", + "me.", + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "de", + "jeito", + "nenhum", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today." + ], + "split_newline": [ + "The customer opened this ticket yesterday and we have asked for a screenshot.", + "The browser console shows no errors and the network tab looks clean to me.", + "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum", + "We will escalate this to the platform team if it is not resolved today." + ], + "count_alpha": 240, + "code_point_length": 297, + "english_rescued_by_words": true + } + }, + { + "name": "portuguese-line-first-then-english", + "state": { + "kind": "string", + "value": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nThe customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nWe will escalate this to the platform team if it is not resolved today." + }, + "state_text": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nThe customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nWe will escalate this to the platform team if it is not resolved today.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 18, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum" + }, + "is_english": false, + "helpers": { + "words": [ + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "de", + "jeito", + "nenhum", + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot", + "The", + "browser", + "console", + "shows", + "no", + "errors", + "and", + "the", + "network", + "tab", + "looks", + "clean", + "to", + "me", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today" + ], + "substitute_identifiers": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nThe customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nWe will escalate this to the platform team if it is not resolved today.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nThe customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nWe will escalate this to the platform team if it is not resolved today.", + "python_lower": "nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nthe customer opened this ticket yesterday and we have asked for a screenshot.\nthe browser console shows no errors and the network tab looks clean to me.\nwe will escalate this to the platform team if it is not resolved today.", + "strip": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nThe customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nWe will escalate this to the platform team if it is not resolved today.", + "split_whitespace": [ + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "de", + "jeito", + "nenhum", + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot.", + "The", + "browser", + "console", + "shows", + "no", + "errors", + "and", + "the", + "network", + "tab", + "looks", + "clean", + "to", + "me.", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today." + ], + "split_newline": [ + "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum", + "The customer opened this ticket yesterday and we have asked for a screenshot.", + "The browser console shows no errors and the network tab looks clean to me.", + "We will escalate this to the platform team if it is not resolved today." + ], + "count_alpha": 240, + "code_point_length": 297, + "english_rescued_by_words": true + } + }, + { + "name": "english-note-with-german-line", + "state": { + "kind": "string", + "value": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nIch kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert\nWe will escalate this to the platform team if it is not resolved today." + }, + "state_text": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nIch kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert\nWe will escalate this to the platform team if it is not resolved today.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 19, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "de", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert" + }, + "is_english": false, + "helpers": { + "words": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot", + "The", + "browser", + "console", + "shows", + "no", + "errors", + "and", + "the", + "network", + "tab", + "looks", + "clean", + "to", + "me", + "Ich", + "kann", + "mich", + "nicht", + "in", + "mein", + "Konto", + "einloggen", + "und", + "das", + "Passwort", + "wird", + "nicht", + "akzeptiert", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today" + ], + "substitute_identifiers": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nIch kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert\nWe will escalate this to the platform team if it is not resolved today.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nIch kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert\nWe will escalate this to the platform team if it is not resolved today.", + "python_lower": "the customer opened this ticket yesterday and we have asked for a screenshot.\nthe browser console shows no errors and the network tab looks clean to me.\nich kann mich nicht in mein konto einloggen und das passwort wird nicht akzeptiert\nwe will escalate this to the platform team if it is not resolved today.", + "strip": "The customer opened this ticket yesterday and we have asked for a screenshot.\nThe browser console shows no errors and the network tab looks clean to me.\nIch kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert\nWe will escalate this to the platform team if it is not resolved today.", + "split_whitespace": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot.", + "The", + "browser", + "console", + "shows", + "no", + "errors", + "and", + "the", + "network", + "tab", + "looks", + "clean", + "to", + "me.", + "Ich", + "kann", + "mich", + "nicht", + "in", + "mein", + "Konto", + "einloggen", + "und", + "das", + "Passwort", + "wird", + "nicht", + "akzeptiert", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today." + ], + "split_newline": [ + "The customer opened this ticket yesterday and we have asked for a screenshot.", + "The browser console shows no errors and the network tab looks clean to me.", + "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert", + "We will escalate this to the platform team if it is not resolved today." + ], + "count_alpha": 250, + "code_point_length": 307, + "english_rescued_by_words": true + } + }, + { + "name": "english-note-with-code-line-and-foreign-line", + "state": { + "kind": "string", + "value": "The customer opened this ticket yesterday and we have asked for a screenshot.\nresult = round(el, 2); os.path.join(a, b)\nNon riesco ad accedere al mio account nel portale e la fattura non arriva\nWe will escalate this to the platform team if it is not resolved today." + }, + "state_text": "The customer opened this ticket yesterday and we have asked for a screenshot.\nresult = round(el, 2); os.path.join(a, b)\nNon riesco ad accedere al mio account nel portale e la fattura non arriva\nWe will escalate this to the platform team if it is not resolved today.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 14, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "it", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": "Non riesco ad accedere al mio account nel portale e la fattura non arriva" + }, + "is_english": false, + "helpers": { + "words": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot", + "result", + "round", + "el", + "os", + "path", + "join", + "a", + "b", + "Non", + "riesco", + "ad", + "accedere", + "al", + "mio", + "account", + "nel", + "portale", + "e", + "la", + "fattura", + "non", + "arriva", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today" + ], + "substitute_identifiers": "The customer opened this ticket yesterday and we have asked for a screenshot.\nresult = round(el, 2); (a, b)\nNon riesco ad accedere al mio account nel portale e la fattura non arriva\nWe will escalate this to the platform team if it is not resolved today.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": true, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The customer opened this ticket yesterday and we have asked for a screenshot.\nresult = round(el, 2); os.path.join(a, b)\nNon riesco ad accedere al mio account nel portale e la fattura non arriva\nWe will escalate this to the platform team if it is not resolved today.", + "python_lower": "the customer opened this ticket yesterday and we have asked for a screenshot.\nresult = round(el, 2); os.path.join(a, b)\nnon riesco ad accedere al mio account nel portale e la fattura non arriva\nwe will escalate this to the platform team if it is not resolved today.", + "strip": "The customer opened this ticket yesterday and we have asked for a screenshot.\nresult = round(el, 2); os.path.join(a, b)\nNon riesco ad accedere al mio account nel portale e la fattura non arriva\nWe will escalate this to the platform team if it is not resolved today.", + "split_whitespace": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot.", + "result", + "=", + "round(el,", + "2);", + "os.path.join(a,", + "b)", + "Non", + "riesco", + "ad", + "accedere", + "al", + "mio", + "account", + "nel", + "portale", + "e", + "la", + "fattura", + "non", + "arriva", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today." + ], + "split_newline": [ + "The customer opened this ticket yesterday and we have asked for a screenshot.", + "result = round(el, 2); os.path.join(a, b)", + "Non riesco ad accedere al mio account nel portale e la fattura non arriva", + "We will escalate this to the platform team if it is not resolved today." + ], + "count_alpha": 206, + "code_point_length": 265, + "english_rescued_by_words": true + } + }, + { + "name": "english-note-with-acronym-line-and-foreign-line", + "state": { + "kind": "string", + "value": "The customer opened this ticket yesterday and we have asked for a screenshot.\nMON LA EST COM DES\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today." + }, + "state_text": "The customer opened this ticket yesterday and we have asked for a screenshot.\nMON LA EST COM DES\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 14, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "fr", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": "MON LA EST COM DES" + }, + "is_english": false, + "helpers": { + "words": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot", + "MON", + "LA", + "EST", + "COM", + "DES", + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "de", + "jeito", + "nenhum", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today" + ], + "substitute_identifiers": "The customer opened this ticket yesterday and we have asked for a screenshot.\nMON LA EST COM DES\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The customer opened this ticket yesterday and we have asked for a screenshot.\n \nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "python_lower": "the customer opened this ticket yesterday and we have asked for a screenshot.\nmon la est com des\nnao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nwe will escalate this to the platform team if it is not resolved today.", + "strip": "The customer opened this ticket yesterday and we have asked for a screenshot.\nMON LA EST COM DES\nNao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\nWe will escalate this to the platform team if it is not resolved today.", + "split_whitespace": [ + "The", + "customer", + "opened", + "this", + "ticket", + "yesterday", + "and", + "we", + "have", + "asked", + "for", + "a", + "screenshot.", + "MON", + "LA", + "EST", + "COM", + "DES", + "Nao", + "consigo", + "entrar", + "na", + "minha", + "conta", + "e", + "a", + "senha", + "nao", + "funciona", + "de", + "jeito", + "nenhum", + "We", + "will", + "escalate", + "this", + "to", + "the", + "platform", + "team", + "if", + "it", + "is", + "not", + "resolved", + "today." + ], + "split_newline": [ + "The customer opened this ticket yesterday and we have asked for a screenshot.", + "MON LA EST COM DES", + "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum", + "We will escalate this to the platform team if it is not resolved today." + ], + "count_alpha": 194, + "code_point_length": 241, + "english_rescued_by_words": true + } + }, + { + "name": "none-state", + "state": { + "kind": "json", + "value": null + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "number-state", + "state": { + "kind": "json", + "value": 42 + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "bool-state", + "state": { + "kind": "json", + "value": true + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "empty-dict", + "state": { + "kind": "json", + "value": {} + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "empty-list", + "state": { + "kind": "json", + "value": [] + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "bytes-valid", + "state": { + "kind": "bytes", + "value": [ + 78, + 195, + 163, + 111, + 32, + 99, + 111, + 110, + 115, + 105, + 103, + 111, + 32, + 101, + 110, + 116, + 114, + 97, + 114, + 32, + 110, + 97, + 32, + 109, + 105, + 110, + 104, + 97, + 32, + 99, + 111, + 110, + 116, + 97, + 32, + 101, + 32, + 97, + 32, + 115, + 101, + 110, + 104, + 97, + 32, + 110, + 195, + 163, + 111, + 32, + 102, + 117, + 110, + 99, + 105, + 111, + 110, + 97 + ] + }, + "state_text": "Não consigo entrar na minha conta e a senha não funciona", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "pt", + "english_hits": 0, + "diacritic_rate": 0.03571428571428571, + "looks_non_english": true + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0357, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false + }, + { + "name": "bytes-invalid", + "state": { + "kind": "bytes", + "value": [ + 255, + 254, + 32, + 110, + 111, + 116, + 32, + 117, + 116, + 102, + 45, + 56, + 32, + 97, + 116, + 32, + 97, + 108, + 108 + ] + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "dict-english-note-and-portuguese-message", + "state": { + "kind": "json", + "value": { + "note": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. ", + "message": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum" + } + }, + "state_text": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 327, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false + }, + { + "name": "dict-portuguese-past-the-cap", + "state": { + "kind": "json", + "value": { + "note": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. ", + "tail": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum" + } + }, + "state_text": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 327, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false + }, + { + "name": "dict-english-name-field", + "state": { + "kind": "json", + "value": { + "name": "José", + "body": "Please reset my password for this account today" + } + }, + "state_text": "José Please reset my password for this account today", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 3, + "diacritic_rate": 0.019230769230769232, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0192, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "dict-cyrillic-name-field", + "state": { + "kind": "json", + "value": { + "name": "Дмитрий", + "body": "Please reset my password for this account today" + } + }, + "state_text": "Дмитрий Please reset my password for this account today", + "detect_script": "latin", + "script_profile": { + "latin": 0.851063829787234, + "cyrillic": 0.14893617021276595 + }, + "latin_profile": { + "language": "en", + "english_hits": 3, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 0.851063829787234, + "cyrillic": 0.14893617021276595 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.1489, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "dict-all-english", + "state": { + "kind": "json", + "value": { + "a": "Please reset my password", + "b": "The dashboard will not load at all" + } + }, + "state_text": "Please reset my password The dashboard will not load at all", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 5, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "nested-list-of-dicts", + "state": { + "kind": "json", + "value": [ + { + "t": "Please reset my password for the account" + }, + { + "t": "Ich kann mich nicht in mein Konto einloggen und das Passwort" + } + ] + }, + "state_text": "Please reset my password for the account Ich kann mich nicht in mein Konto einloggen und das Passwort", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "de", + "english_hits": 4, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "de", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false + }, + { + "name": "deep-nesting-within-limit", + "state": { + "kind": "json", + "value": [ + [ + [ + [ + [ + [ + "Ich kann mich nicht in mein Konto einloggen und das Passwort" + ] + ] + ] + ] + ] + ] + }, + "state_text": "Ich kann mich nicht in mein Konto einloggen und das Passwort", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "de", + "english_hits": 1, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "de", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false + }, + { + "name": "deep-nesting-past-limit", + "state": { + "kind": "json", + "value": [ + [ + [ + [ + [ + [ + [ + "Ich kann mich nicht in mein Konto einloggen und das Passwort" + ] + ] + ] + ] + ] + ] + ] + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "dict-with-korean-value", + "state": { + "kind": "json", + "value": { + "id": "ORDER-1", + "msg": "계정에 로그인할 수 없어요 도와주세요" + } + }, + "state_text": "ORDER-1 계정에 로그인할 수 없어요 도와주세요", + "detect_script": "hangul", + "script_profile": { + "latin": 0.23809523809523808, + "hangul": 0.7619047619047619 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "hangul", + "script_profile": { + "latin": 0.23809523809523808, + "hangul": 0.7619047619047619 + }, + "language": null, + "is_english": false, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.7619, + "mixed_segment": null + }, + "is_english": false + }, + { + "name": "dict-with-code-leaf", + "state": { + "kind": "json", + "value": { + "trace": "result = round(el, 2);\nos.path.join(a, b)", + "msg": "Please take a look at this when you can" + } + }, + "state_text": "result = round(el, 2);\nos.path.join(a, b) Please take a look at this when you can", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 5, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "list-of-numbers", + "state": { + "kind": "json", + "value": [ + 1, + 2.5, + null, + true + ] + }, + "state_text": "", + "detect_script": "unknown", + "script_profile": {}, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "unknown", + "script_profile": {}, + "language": null, + "is_english": true, + "language_undecided": true, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true + }, + { + "name": "long-single-leaf-over-budget", + "state": { + "kind": "string", + "value": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. " + }, + "state_text": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 327, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "en", + "is_english": true, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": true, + "helpers": { + "words": [ + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output" + ], + "substitute_identifiers": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. ", + "non_latin_words": [], + "named_prose_language": null, + "has_code_line": false, + "has_joined_tokens": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "blank_upper_runs": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. ", + "python_lower": "the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. the customer reports that the dashboard will not load after the update. we have asked for a screenshot and the browser console output. ", + "strip": "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output.", + "split_whitespace": [ + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output.", + "The", + "customer", + "reports", + "that", + "the", + "dashboard", + "will", + "not", + "load", + "after", + "the", + "update.", + "We", + "have", + "asked", + "for", + "a", + "screenshot", + "and", + "the", + "browser", + "console", + "output." + ], + "split_newline": [ + "The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. The customer reports that the dashboard will not load after the update. We have asked for a screenshot and the browser console output. " + ], + "count_alpha": 4400, + "code_point_length": 5400, + "english_rescued_by_words": true + } + }, + { + "name": "two-leaves-budget-split", + "state": { + "kind": "json", + "value": [ + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "Nao consigo entrar na minha conta e a senha nao funciona" + ] + }, + "state_text": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Nao consi", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": null, + "english_hits": 0, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "pt", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": null + }, + "is_english": false + }, + { + "name": "dict-short-english-note-and-german-message", + "state": { + "kind": "json", + "value": { + "note": "The customer opened this ticket yesterday and we have asked for a screenshot. The browser console shows no errors and the network tab looks clean to me. We will escalate this to the platform team today. ", + "message": "Ich kann mich nicht in mein Konto einloggen und das Passwort geht nicht" + } + }, + "state_text": "The customer opened this ticket yesterday and we have asked for a screenshot. The browser console shows no errors and the network tab looks clean to me. We will escalate this to the platform team today. Ich kann mich nicht in mein Konto einloggen und das Passwort geht nicht", + "detect_script": "latin", + "script_profile": { + "latin": 1.0 + }, + "latin_profile": { + "language": "en", + "english_hits": 16, + "diacritic_rate": 0.0, + "looks_non_english": false + }, + "analyse": { + "script": "latin", + "script_profile": { + "latin": 1.0 + }, + "language": "de", + "is_english": false, + "language_undecided": false, + "diacritic_rate": 0.0, + "non_latin_fraction": 0.0, + "mixed_segment": "Ich kann mich nicht in mein Konto einloggen und das Passwort geht nicht" + }, + "is_english": false + } + ] +} diff --git a/laya-java/fixtures/lang_tables.json b/laya-java/fixtures/lang_tables.json index 396739dac..e4a732f7f 100644 --- a/laya-java/fixtures/lang_tables.json +++ b/laya-java/fixtures/lang_tables.json @@ -282,6 +282,19 @@ ] } ], + "stop_word_order": [ + "en", + "fr", + "de", + "es", + "pt", + "it", + "nl", + "sv", + "ro", + "bn", + "az" + ], "stop_words": { "az": [ "amma", diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java new file mode 100644 index 000000000..7e31ee7bb --- /dev/null +++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java @@ -0,0 +1,1030 @@ +package com.convaiinnovations.laya.lang; + +import com.convaiinnovations.laya.decode.Rounding; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Script and language detection, used to route a state between the English and multilingual + * checkpoints. A port of {@code laya/lang.py}. + * + *

Routing needs one decision: is this English Latin-script text, or is it something the + * English checkpoint cannot read? Script detection is exact. The Latin-script language guess + * is a stopword and diacritic heuristic and is explicitly best-effort -- pass an explicit model or + * language when you already know it. + * + *

Why the regexes are hand-written scanners. The reference uses four patterns, and + * {@link java.util.regex} is not a drop-in for any of them. Java's {@code \\w} is a different set + * from Python's (it admits combining marks and join controls), its {@code \\d} and {@code \\p{L}} + * follow the JDK's Unicode version rather than CPython's, and {@code String.split} drops trailing + * empty fields where Python's keeps them. Each pattern is therefore written out as a scan over + * {@link UnicodeTables}, whose predicates are recorded from CPython. The identifier scan also + * reproduces the reference's lookbehind, which is what keeps it linear: without it the pattern + * retries at every offset inside a run of word characters, and a state is user input. + */ +public final class LanguageDetection { + + /** How much of a state detection reads, matching the reference's {@code max_chars}. */ + public static final int MAX_CHARS = 4000; + + /** Nesting depth beyond which a state is not descended into. */ + private static final int MAX_DEPTH = 6; + + /** Below this code point every letter is Latin, so script lookup can be skipped. */ + private static final int LATIN_BELOW_COUNTS = 0x02B0; + + /** + * The same cut for {@code scriptOf}, which the reference sets lower than the one above. + * + *

The two differ on U+0250..U+02AF, the IPA extensions: a letter there is counted as Latin + * but names no script, so it can neither start nor extend a non-Latin word run. Kept as two + * constants rather than unified, because unifying them would change which runs + * {@code nonLatinWords} reports. + */ + private static final int LATIN_BELOW_SCRIPT_OF = 0x0250; + + /** The tally slot every letter no named script claims is counted under. */ + private static final int OTHER_SLOT = LanguageTables.SCRIPT_COUNT; + + /** Non-English Latin letters, sorted for binary search; the set itself is a generated table. */ + private static final int[] DIACRITICS = sortedCodePoints(LanguageTables.NON_EN_DIACRITICS); + + /** The Swedish phrase openers {@code latinProfile} accepts a short login request on. */ + private static final Set SWEDISH_PHRASE_OPENERS = Set.of("kan", "jag", "vi"); + + private LanguageDetection() { + } + + /** The evidence behind the Latin-script language guess. {@code language} is null when undecided. */ + public record LatinProfile(String language, int englishHits, double diacriticRate, + boolean looksNonEnglish) { + } + + /** + * A full detection result. + * + * @param script dominant script: {@code latin}, {@code han}, ... or {@code unknown} + * @param scriptProfile fraction of alphabetic characters per script, Latin first + * @param language best-effort language code, or null when undecided + * @param english whether the English checkpoint can be expected to read this + * @param languageUndecided whether the language could not be named + * @param diacriticRate share of characters that are non-English Latin letters + * @param nonLatinFraction share of letters outside Latin + * @param mixedSegment the line or field that made a mostly English state non-English + */ + public record Analysis(String script, Map scriptProfile, String language, + boolean english, boolean languageUndecided, double diacriticRate, + double nonLatinFraction, String mixedSegment) { + } + + // ------------------------------------------------------------------ state flattening + + /** + * The string leaves of a state, which is what detection reads. + * + *

Keys are ignored: they are usually English field names. {@code byte[]} stands for the + * reference's {@code bytes} and is decoded strictly, so an undecodable leaf contributes + * nothing rather than mojibake. A {@code Set} is deliberately not descended into even though + * it is a {@code Collection}: its iteration order is unspecified, and detection must not + * depend on it. + */ + private static List iterText(Object state) { + List out = new ArrayList<>(); + collectText(state, 0, out); + return out; + } + + private static void collectText(Object state, int depth, List out) { + if (depth > MAX_DEPTH || state == null) { + return; + } + if (state instanceof String) { + out.add((String) state); + } else if (state instanceof byte[]) { + String decoded = decodeUtf8((byte[]) state); + if (decoded != null) { + out.add(decoded); + } + } else if (state instanceof Map) { + for (Object value : ((Map) state).values()) { + collectText(value, depth + 1, out); + } + } else if (state instanceof List) { + for (Object value : (List) state) { + collectText(value, depth + 1, out); + } + } + } + + /** Strict UTF-8, or null for what the reference catches as {@code UnicodeDecodeError}. */ + private static String decodeUtf8(byte[] bytes) { + CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + try { + return decoder.decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + return null; + } + } + + /** Flatten a state into the text used for detection. */ + public static String stateText(Object state) { + return stateText(state, MAX_CHARS); + } + + /** Flatten a state into at most {@code maxChars} code points of text. */ + public static String stateText(Object state, int maxChars) { + List parts = new ArrayList<>(); + int budget = maxChars; + for (String leaf : iterText(state)) { + if (budget <= 0) { + break; + } + int length = codePointLength(leaf); + if (length > budget) { + parts.add(head(leaf, budget)); + break; + } + parts.add(leaf); + // Account for the joining space without materialising the full text first. + budget -= length + 1; + } + return head(String.join(" ", parts), maxChars); + } + + // ------------------------------------------------------------------ scripts + + /** + * Count the alphabetic characters of {@code text} by script, in one pass. + * + *

Latin is inserted last so that {@code scriptFromCounts} keeps the tie-break: a named + * script wins a tie against Latin, because the maximum is the first of equal values and Latin + * is the last key. Both the dominant script and the per-script fractions are read off one + * pass. + */ + static Map scriptCounts(String text) { + // Tallied into an int[] and named once at the end, rather than a map update per letter: + // the reference hashes a string per non-Latin character, and so does laya-ts. + int[] tally = new int[OTHER_SLOT + 1]; + int[] firstSeen = new int[OTHER_SLOT + 1]; + int distinct = 0; + int latin = 0; + int i = 0; + while (i < text.length()) { + int cp = text.codePointAt(i); + i += Character.charCount(cp); + if (!UnicodeTables.isAlpha(cp)) { + continue; + } + if (isLatinRange(cp, LATIN_BELOW_COUNTS)) { + latin++; + continue; + } + int index = LanguageTables.lookupScriptIndex(cp); + // An alphabetic character no range claims is counted under "other" rather than + // nowhere: text written only in an unlisted script would otherwise total zero, read + // as "unknown", and be handed to the English checkpoint that has no tokens for it. + int slot = index < 0 ? OTHER_SLOT : index; + if (tally[slot]++ == 0) { + firstSeen[distinct++] = slot; + } + } + // Insertion order is first-encounter order in the text, which is what the reference's + // dict gives and what `profileFromCounts` carries through to the public profile. + Map counts = new LinkedHashMap<>(); + for (int n = 0; n < distinct; n++) { + int slot = firstSeen[n]; + counts.put(slot == OTHER_SLOT ? "other" : LanguageTables.scriptName(slot), tally[slot]); + } + counts.put("latin", latin); + return counts; + } + + /** Latin, IPA extensions, Latin Extended-Additional and the fullwidth Latin letters. */ + private static boolean isLatinRange(int cp, int below) { + return cp < below + || (cp >= 0x1E00 && cp <= 0x1EFF) + || (cp >= 0xFF21 && cp <= 0xFF3A) + || (cp >= 0xFF41 && cp <= 0xFF5A); + } + + static String scriptFromCounts(Map counts) { + String best = null; + int bestCount = 0; + for (Map.Entry entry : counts.entrySet()) { + if (best == null || entry.getValue() > bestCount) { + best = entry.getKey(); + bestCount = entry.getValue(); + } + } + return bestCount == 0 ? "unknown" : best; + } + + static Map profileFromCounts(Map counts) { + int total = 0; + for (int value : counts.values()) { + total += value; + } + if (total == 0) { + return Map.of(); + } + // Latin first in the public mapping, the order callers saw before this was refactored. + Map profile = new LinkedHashMap<>(); + Integer latin = counts.get("latin"); + if (latin != null && latin != 0) { + profile.put("latin", latin / (double) total); + } + for (Map.Entry entry : counts.entrySet()) { + if (!"latin".equals(entry.getKey()) && entry.getValue() != 0) { + profile.put(entry.getKey(), entry.getValue() / (double) total); + } + } + return Collections.unmodifiableMap(profile); + } + + /** Dominant script of {@code text}, or {@code unknown} when it holds no letters. */ + public static String detectScript(String text) { + return scriptFromCounts(scriptCounts(text)); + } + + /** Fraction of alphabetic characters belonging to each detected script. */ + public static Map scriptProfile(String text) { + return profileFromCounts(scriptCounts(text)); + } + + /** The named non-Latin script of one letter, or null for Latin and for unclaimed letters. */ + static String scriptOf(int cp) { + int index = scriptIndexOf(cp); + return index < 0 ? null : LanguageTables.scriptName(index); + } + + /** + * {@link #scriptOf} as an index, which is what the run scan compares. + * + *

{@code scriptOf} delegates here rather than the other way round, so there is one + * implementation of the rule and the two cannot drift. + */ + private static int scriptIndexOf(int cp) { + if (isLatinRange(cp, LATIN_BELOW_SCRIPT_OF)) { + return -1; + } + return LanguageTables.lookupScriptIndex(cp); + } + + /** + * Non-Latin runs that read as words rather than as annotation inside English prose. + * + *

English prose carries three kinds of non-Latin letter that are not a request written in + * another script, and each is excluded here: a symbol ({@code Set alpha to 0.05}, one letter), + * a proper name (capitalised), and a pronunciation in IPA, which no script range claims. A + * combining mark belongs to the letter before it and never splits a word, so an accented + * Cyrillic name stays one capitalised word rather than becoming two. + */ + static List nonLatinWords(String text) { + List runs = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + int script = -1; + int i = 0; + while (i < text.length()) { + int cp = text.codePointAt(i); + i += Character.charCount(cp); + if (UnicodeTables.isCombining(cp)) { + continue; + } + int here = scriptIndexOf(cp); + if (here >= 0 && here == script) { + current.appendCodePoint(cp); + continue; + } + if (current.length() > 0) { + runs.add(current.toString()); + } + current = new StringBuilder(); + if (here >= 0) { + current.appendCodePoint(cp); + } + script = here; + } + if (current.length() > 0) { + runs.add(current.toString()); + } + List out = new ArrayList<>(); + for (String run : runs) { + if (codePointLength(run) >= 2 && !UnicodeTables.isUpper(run.codePointAt(0))) { + out.add(run); + } + } + return out; + } + + // ------------------------------------------------------------------ the four patterns + + /** The reference's {@code [^\\W\\d_]+}: maximal runs of word characters that are not digits. */ + static List words(String text) { + List out = new ArrayList<>(); + int i = 0; + int length = text.length(); + while (i < length) { + int cp = text.codePointAt(i); + int width = Character.charCount(cp); + if (!UnicodeTables.isWordChar(cp)) { + i += width; + continue; + } + int start = i; + while (i < length) { + int inner = text.codePointAt(i); + if (!UnicodeTables.isWordChar(inner)) { + break; + } + i += Character.charCount(inner); + } + out.add(text.substring(start, i)); + } + return out; + } + + /** A character of the reference's {@code [\\w-]}. */ + private static boolean isIdentifierChar(int cp) { + return cp == '-' || UnicodeTables.isPythonWordChar(cp); + } + + /** + * The reference's {@code (?A token whose dot or at-sign joins word characters is an identifier, not prose: + * {@code github.com}, {@code user@acme.com}, {@code v1.2.3}, {@code U.S.A.}. The word scan + * splits them into pieces that collide with real function words -- {@code com} is Portuguese + * for "with", {@code o} is its article -- so a state that was mostly links scored a language + * it does not contain. A sentence-final period keeps its word: there must be word characters + * on both sides of the dot. + * + *

No backtracking is needed, which is what makes this linear. {@code [.@]} and + * {@code [\\w-]} are disjoint, so once a maximal run of identifier characters is consumed the + * next character either is a joiner or is not, and giving a character back cannot help. The + * lookbehind means a match can only begin where a run begins. + */ + static String substituteIdentifiers(String text) { + StringBuilder out = null; + int i = 0; + int length = text.length(); + int copied = 0; + while (i < length) { + int cp = text.codePointAt(i); + int width = Character.charCount(cp); + boolean atRunStart = i == 0 || !isIdentifierChar(text.codePointBefore(i)); + if (!atRunStart) { + i += width; + continue; + } + int end = matchIdentifier(text, i); + if (end < 0) { + i += width; + continue; + } + if (out == null) { + out = new StringBuilder(length); + } + out.append(text, copied, i).append(' '); + i = end; + copied = end; + } + if (out == null) { + return text; + } + return out.append(text, copied, length).toString(); + } + + /** End offset of an identifier match starting at {@code from}, or -1 when there is none. */ + private static int matchIdentifier(String text, int from) { + int i = runEnd(text, from); + int groups = 0; + while (i < text.length()) { + char joiner = text.charAt(i); + if (joiner != '.' && joiner != '@') { + break; + } + int after = runEnd(text, i + 1); + if (after == i + 1) { + break; // the joiner is not followed by [\w-]+ + } + i = after; + groups++; + } + return groups > 0 ? i : -1; + } + + /** End of the maximal run of identifier characters starting at {@code from}. */ + private static int runEnd(String text, int from) { + int i = from; + while (i < text.length()) { + int cp = text.codePointAt(i); + if (!isIdentifierChar(cp)) { + break; + } + i += Character.charCount(cp); + } + return i; + } + + /** + * The reference's {@code [=;{}\\[\\]]|\\w\\(}: does this line carry code syntax? + * + *

Code is not prose in any language, but split into words it reads as one: {@code os.path} + * is Portuguese, {@code round(el, 2)} Spanish. Prose keeps "Deu erro (500)": the parenthesis + * follows a space rather than a word character. + */ + static boolean hasCodeLine(String text) { + for (int i = 0; i < text.length(); i++) { + char ch = text.charAt(i); + if (ch == '=' || ch == ';' || ch == '{' || ch == '}' || ch == '[' || ch == ']') { + return true; + } + if (ch == '(' && i > 0 && UnicodeTables.isPythonWordChar(text.codePointBefore(i))) { + return true; + } + } + return false; + } + + /** + * The reference's {@code [^\\W_][._/\\\\][^\\W_]}: is this token a compound name? + * + *

{@code Nav/Com} and {@code OS/2} read as Portuguese, {@code C:\\DOS\\mode} too. A token + * holding a letter or digit, a joiner, and another letter or digit is dropped whole. + */ + static boolean hasJoined(String token) { + int i = 0; + int length = token.length(); + while (i < length) { + int cp = token.codePointAt(i); + int width = Character.charCount(cp); + if ((cp == '.' || cp == '_' || cp == '/' || cp == '\\') + && i > 0 && i + width < length + && UnicodeTables.isWordOrDigit(token.codePointBefore(i)) + && UnicodeTables.isWordOrDigit(token.codePointAt(i + width))) { + return true; + } + i += width; + } + return false; + } + + /** + * The reference's {@code [^\\W\\d_]{2,}} substitution that blanks all-caps runs. + * + *

An all-caps token inside mixed-case text is an acronym or a code: {@code MON}, {@code LA}, + * {@code EST}, {@code COM}, {@code DES} are hockey teams, states, time zones and radio bands, + * not French or Portuguese. A segment written entirely in capitals keeps its words -- a + * customer shouting in Portuguese is still Portuguese -- which is why the caller only applies + * this when the segment holds a lowercase letter. + */ + static String blankUpperRuns(String text) { + StringBuilder out = new StringBuilder(text.length()); + int i = 0; + int length = text.length(); + while (i < length) { + int cp = text.codePointAt(i); + int width = Character.charCount(cp); + if (!UnicodeTables.isWordChar(cp)) { + out.appendCodePoint(cp); + i += width; + continue; + } + int start = i; + while (i < length) { + int inner = text.codePointAt(i); + if (!UnicodeTables.isWordChar(inner)) { + break; + } + i += Character.charCount(inner); + } + String run = text.substring(start, i); + if (codePointLength(run) >= 2 && UnicodeTables.isUpperString(run)) { + out.append(' '); + } else { + out.append(run); + } + } + return out.toString(); + } + + // ------------------------------------------------------------------ the Latin guess + + /** + * Whether plain-English function words outvote a marginal diacritic rate. + * + *

The rate is measured over every character, so one accented loanword or proper noun in a + * short sentence clears the floor outright. English still wins when it shows at least two + * distinct function words no other list holds and at most one word carrying a non-English + * letter: one loanword is not a non-English vocabulary, while the odd word a Danish or Swedish + * sentence picks up is not an English sentence either. A rate well above the rescue rate + * vetoes regardless. + */ + static boolean englishRescuedByWords(List words, double diacriticRate) { + if (diacriticRate >= LanguageTables.ENGLISH_RESCUE_DIACRITIC_RATE) { + return false; + } + Set unique = new LinkedHashSet<>(words); + int englishOnly = 0; + for (String word : unique) { + if (LanguageTables.EN_ONLY_WORDS.contains(word)) { + englishOnly++; + } + } + if (englishOnly < 2) { + return false; + } + int withDiacritic = 0; + for (String word : unique) { + if (hasDiacritic(word)) { + withDiacritic++; + } + } + return withDiacritic <= 1; + } + + /** + * The evidence behind the Latin-script language guess. + * + *

A non-English language is only named when it matched at least one word that no other list + * claims: shared function words alone identify no particular language. "Undecided" and + * "English" are different answers and only one of them is safe to send to the English + * checkpoint, which is why the evidence is returned and not just the verdict. + */ + public static LatinProfile latinProfile(String text) { + // U+0130 lowercases to i plus a combining dot, which matches no word list. + List words = words(pythonLower(substituteIdentifiers(text).replace("\u0130", "i"))); + String lowered = pythonLower(text); + int diacritics = 0; + int i = 0; + while (i < lowered.length()) { + int cp = lowered.codePointAt(i); + i += Character.charCount(cp); + if (isDiacritic(cp)) { + diacritics++; + } + } + double diacriticRate = diacritics / (double) Math.max(1, codePointLength(lowered)); + boolean nonEnglish = diacriticRate >= LanguageTables.NON_EN_DIACRITIC_RATE; + Set unique = new LinkedHashSet<>(words); + boolean nordicOverlap = intersects(unique, LanguageTables.NORDIC_OVERLAP_WORDS) + && !intersects(unique, LanguageTables.EN_ONLY_WORDS); + + if (words.size() > 1 && words.size() < 4 + && intersects(unique, LanguageTables.SHORT_SWEDISH_WORDS)) { + return new LatinProfile("sv", 0, diacriticRate, nonEnglish); + } + if (words.size() < 4) { + return new LatinProfile(null, 0, diacriticRate, nonEnglish || nordicOverlap); + } + + // A collision word counts once however often it repeats; every other word counts its hits. + Map counts = new LinkedHashMap<>(); + for (String word : words) { + counts.merge(word, 1, Integer::sum); + } + Map scores = new LinkedHashMap<>(); + for (Map.Entry> entry : LanguageTables.STOP_WORDS.entrySet()) { + Set stop = entry.getValue(); + int score = 0; + for (Map.Entry seen : counts.entrySet()) { + if (stop.contains(seen.getKey())) { + score += LanguageTables.EN_COLLISION_WORDS.contains(seen.getKey()) + ? 1 : seen.getValue(); + } + } + scores.put(entry.getKey(), score); + } + int english = scores.getOrDefault("en", 0); + + // Only a language that matched at least one word no other list claims may be named. + // Without that condition the top score can be pure overlap -- `la` and `e` in Romanian + // text made Italian the winner -- which is a guess dressed as a detection. Such a + // language is dropped from the running rather than merely losing the tie, so a lesser + // score with real evidence still gets named, and the text stays undecided when no list + // has any. + String bestLanguage = null; + int best = 0; + boolean seenAny = false; + for (Map.Entry entry : scores.entrySet()) { + String language = entry.getKey(); + if ("en".equals(language) || !hasOwnEvidence(unique, language)) { + continue; + } + if (!seenAny || entry.getValue() > best) { + bestLanguage = language; + best = entry.getValue(); + seenAny = true; + } + } + + String language = null; + if (bestLanguage != null && best >= Math.max(2, english + 2)) { + // a non-English language needs a clear margin over English function words + language = bestLanguage; + } else if ("sv".equals(bestLanguage) && words.contains("inte") && words.contains("kan") + && SWEDISH_PHRASE_OPENERS.contains(words.get(0)) && english <= 1) { + // Short login requests such as "kan inte logga in" carry a distinctive Swedish phrase + // but also one English-shaped token (`in`). Do not treat `kan` alone as Swedish: it is + // common in Danish and Norwegian too. + language = bestLanguage; + } else if (bestLanguage != null && nonEnglish && best >= Math.max(2, english)) { + // Needs two hits here too. One shared function word ("para" in Turkish text) named + // Spanish on the strength of the diacritics alone, which is a guess dressed as a + // detection. + language = bestLanguage; + } else if (english != 0 + && (!nonEnglish || englishRescuedByWords(words, diacriticRate))) { + language = "en"; + } + boolean looksNonEnglish = nonEnglish || (language == null && nordicOverlap); + return new LatinProfile(language, english, diacriticRate, looksNonEnglish); + } + + /** Whether {@code language} matched a word of its own, rather than only shared ones. */ + private static boolean hasOwnEvidence(Set words, String language) { + Set stop = LanguageTables.STOP_WORDS.get(language); + if (stop == null) { + return false; + } + for (String word : words) { + if (stop.contains(word) && !LanguageTables.SHARED_WORDS.contains(word)) { + return true; + } + } + return false; + } + + /** Best-effort language code for Latin-script text, or null when undecided. */ + public static String guessLatinLanguage(String text) { + return latinProfile(text).language(); + } + + /** + * Language code for one non-code line, or null when it does not name a foreign language. + * + *

Same evidence bar as the segment scan: four words, a language the Latin guess will name, + * and two different words of that language. Acronyms and slash compounds are not + * words. + */ + static String namedProseLanguage(String segment) { + if (UnicodeTables.isBlank(segment) || hasCodeLine(segment)) { + return null; + } + List kept = new ArrayList<>(); + for (String token : UnicodeTables.splitOnWhitespace(segment)) { + if (!hasJoined(token)) { + kept.add(token); + } + } + String prose = String.join(" ", kept); + if (UnicodeTables.hasLower(prose)) { + prose = blankUpperRuns(prose); + } + List tokens = words(prose); + if (tokens.size() < 4) { + return null; + } + String language = latinProfile(prose).language(); + if (language == null || "en".equals(language)) { + return null; + } + Set stop = LanguageTables.STOP_WORDS.get(language); + if (stop == null) { + return null; + } + Set distinct = new LinkedHashSet<>(); + for (String token : tokens) { + String lowered = pythonLower(token); + if (stop.contains(lowered)) { + distinct.add(lowered); + } + } + return distinct.size() < 2 ? null : language; + } + + /** + * The first line or field that, read on its own, is named a non-English language. + * + *

A segment needs the evidence a whole state needs -- at least four words, and a language + * the Latin guess names -- and, because one line carries far less text than a state, two + * things more: the words that name the language must be two different ones ("COM ... COM" in + * an English radio listing is one word seen twice), and acronyms and slash compounds are not + * words. This adds no new way to call English text foreign; it only stops a longer English + * part from outvoting a foreign one. Reads at most {@code maxChars} characters in all. + * + * @return the language and the segment, or null + */ + static String[] nonEnglishSegment(Object state, int maxChars) { + int seen = 0; + for (String leaf : iterText(state)) { + for (String line : leaf.split("\n", -1)) { + if (seen >= maxChars) { + return null; + } + String segment = head(line, maxChars - seen); + seen += codePointLength(segment); + String language = namedProseLanguage(segment); + if (language != null) { + return new String[] {language, UnicodeTables.strip(segment)}; + } + } + } + return null; + } + + // ------------------------------------------------------------------ the verdict + + /** + * Detection result for one already-flattened string. + * + *

Does not look for a foreign line inside mostly-English text; {@link #analyse} does that, + * because it needs the original state and not only the joined window. + */ + static Analysis analyseText(String text) { + Map counts = scriptCounts(text); + Map profile = profileFromCounts(counts); + String script = scriptFromCounts(counts); + double nonLatin = profile.isEmpty() + ? 0.0 + : Rounding.round4(1.0 - profile.getOrDefault("latin", 0.0)); + int nonLatinLetters = roundHalfEven(nonLatin * countAlpha(text)); + if ("latin".equals(script) && !nonLatinWords(text).isEmpty() + && (nonLatin >= LanguageTables.NON_LATIN_FRACTION + || (nonLatin >= LanguageTables.NON_LATIN_MIN_FRACTION + && nonLatinLetters >= LanguageTables.NON_LATIN_MIN_LETTERS))) { + // Non-Latin text is not for the English checkpoint even when Latin letters are the + // plurality: a brand name or order code outvotes the CJK request around it letter for + // letter, though one CJK character carries far more than a letter. + script = dominantNonLatin(profile); + } + if ("unknown".equals(script)) { + return new Analysis("unknown", profile, null, true, true, 0.0, 0.0, null); + } + if (!"latin".equals(script)) { + return new Analysis(script, profile, null, false, true, 0.0, nonLatin, null); + } + LatinProfile latin = latinProfile(text); + // Undecided is not English. Treating it as English sent every Latin-script language we + // hold no stopwords for to the checkpoint that cannot read it, silently. When nothing + // identifies the language, non-English letters or a shared Swedish-Danish marker can + // still prefer the multilingual checkpoint; text with neither signal still goes to the + // English one. + boolean undecided = latin.language() == null; + boolean english = "en".equals(latin.language()) || (undecided && !latin.looksNonEnglish()); + return new Analysis("latin", profile, latin.language(), english, undecided, + Rounding.round4(latin.diacriticRate()), nonLatin, null); + } + + /** + * The non-Latin script with the largest share. + * + *

Only reached when the non-Latin fraction is at least + * {@link LanguageTables#NON_LATIN_MIN_FRACTION}, which cannot hold unless the profile has a + * non-Latin key -- so the fallback is unreachable, and the reference raises there too. + */ + private static String dominantNonLatin(Map profile) { + String best = null; + double bestShare = 0.0; + for (Map.Entry entry : profile.entrySet()) { + if ("latin".equals(entry.getKey())) { + continue; + } + if (best == null || entry.getValue() > bestShare) { + best = entry.getKey(); + bestShare = entry.getValue(); + } + } + if (best == null) { + throw new IllegalStateException("no non-Latin script in a profile that is not all Latin"); + } + return best; + } + + /** + * A string value that is itself not safe for the English checkpoint, else null. + * + *

A one-word name and a capitalised non-Latin name stay out: the same rules the Latin + * guess and the non-Latin word scan already use, so a name field cannot pull an English + * ticket onto the multilingual checkpoint. Code lines, acronyms and slash compounds stay out + * too, so a pasted traceback is not a message. Each line is capped on its own, so a long + * earlier field does not consume the budget of the next one. + */ + static Analysis leafNonEnglish(String leaf) { + int bestLetters = -1; + Analysis best = null; + for (String line : leaf.split("\n", -1)) { + // A line this short cannot be selected, so skip it before the slice, the blank check, + // the code-line scan and a full pass. Each branch below needs four word tokens or + // NON_LATIN_MIN_LETTERS letters. The word scan matches maximal runs of letters and + // letter-like numerals, so four tokens need three separators between them: seven + // characters, and ten letters need ten. What makes seven safe rather than six is that + // every one of those counts is taken on the raw line -- U+0130 lowers to two code + // points, so counting on the lowered line would let a four-character line reach four + // tokens, which is why the Latin guess replaces U+0130 before lowering. + if (codePointLength(line) < 7) { + continue; + } + String sample = head(line, MAX_CHARS); + if (UnicodeTables.isBlank(sample) || hasCodeLine(sample)) { + continue; + } + Analysis detected = analyseText(sample); + if (detected.english()) { + continue; + } + String language = detected.language(); + if (language != null && !"en".equals(language)) { + if (namedProseLanguage(sample) == null) { + continue; + } + } else if (!"latin".equals(detected.script()) && !"unknown".equals(detected.script())) { + if (nonLatinWords(sample).isEmpty() + || countAlpha(sample) < LanguageTables.NON_LATIN_MIN_LETTERS) { + continue; + } + } else if (!(detected.languageUndecided() + && detected.diacriticRate() >= LanguageTables.NON_EN_DIACRITIC_RATE + && words(sample).size() >= 4)) { + continue; + } + int letters = countAlpha(sample); + if (letters > bestLetters) { + bestLetters = letters; + best = detected; + } + } + return best; + } + + /** + * Full detection result for a state. + * + *

String values are what get read. When a state has several of them, one non-English value + * is enough: joining every value into one window let a long English note fill the window, or + * outvote a short German message, and that message was then sent to the English checkpoint. + * The segment scan stops at {@link #MAX_CHARS}, which is what keeps a huge field cheap; a + * value it did not reach is read on its own afterwards. + */ + public static Analysis analyse(Object state) { + Analysis result = analyseText(stateText(state)); + if ("latin".equals(result.script()) && result.english()) { + // A Portuguese ticket with an English stack trace, error payload or form template + // reads as English as a whole, because the English part is longer -- yet the part a + // question is about is the customer's, and the English checkpoint cannot read it. The + // cost is lopsided: English sent to multilingual loses a few points, the reverse + // loses calibration. So a state that would go to English is checked line by line and + // field by field. + List leaves = iterText(state); + boolean multiline = false; + for (String leaf : leaves) { + if (leaf.indexOf('\n') >= 0) { + multiline = true; + break; + } + } + // a single line has no other part to be outvoted by, and was just read whole + if (leaves.size() > 1 || multiline) { + String[] found = nonEnglishSegment(state, MAX_CHARS); + if (found != null) { + result = new Analysis(result.script(), result.scriptProfile(), found[0], + false, false, result.diacriticRate(), result.nonLatinFraction(), + found[1]); + } + } + } + // A plain string was just read whole. A structured state can still hide a message past + // the segment cap, or in a script the Latin guess does not name. + if (state == null || state instanceof String || state instanceof byte[] + || !result.english()) { + return result; + } + int bestLetters = -1; + Analysis best = null; + for (String leaf : iterText(state)) { + Analysis detected = leafNonEnglish(leaf); + if (detected == null) { + continue; + } + int letters = countAlpha(head(leaf, MAX_CHARS)); + if (letters > bestLetters) { + bestLetters = letters; + best = detected; + } + } + if (best == null) { + return result; + } + return new Analysis(result.script(), result.scriptProfile(), best.language(), false, + best.languageUndecided(), result.diacriticRate(), result.nonLatinFraction(), + result.mixedSegment()); + } + + /** Whether the English checkpoint can be expected to read this state. */ + public static boolean isEnglish(Object state) { + return analyse(state).english(); + } + + // ------------------------------------------------------------------ small helpers + + private static String pythonLower(String text) { + return UnicodeTables.pythonLower(text); + } + + private static boolean intersects(Set words, Set other) { + for (String word : words) { + if (other.contains(word)) { + return true; + } + } + return false; + } + + private static boolean hasDiacritic(String word) { + int i = 0; + while (i < word.length()) { + int cp = word.codePointAt(i); + i += Character.charCount(cp); + if (isDiacritic(cp)) { + return true; + } + } + return false; + } + + private static boolean isDiacritic(int codePoint) { + return Arrays.binarySearch(DIACRITICS, codePoint) >= 0; + } + + /** The generated set as a sorted array, so membership needs no boxing per character. */ + private static int[] sortedCodePoints(Set characters) { + int[] out = new int[characters.size()]; + int at = 0; + for (String character : characters) { + out[at++] = character.codePointAt(0); + } + Arrays.sort(out); + return out; + } + + static int countAlpha(String text) { + int count = 0; + int i = 0; + while (i < text.length()) { + int cp = text.codePointAt(i); + i += Character.charCount(cp); + if (UnicodeTables.isAlpha(cp)) { + count++; + } + } + return count; + } + + private static int codePointLength(String text) { + return text.codePointCount(0, text.length()); + } + + /** The first {@code count} code points of {@code text}, which is Python's slice. */ + private static String head(String text, int count) { + if (count <= 0) { + return ""; + } + if (codePointLength(text) <= count) { + return text; + } + return text.substring(0, text.offsetByCodePoints(0, count)); + } + + /** + * Python's one-argument {@code round}: half to even on the exact binary value. + * + *

{@link Math#rint} is exactly that rule -- IEEE round-to-nearest, ties-to-even -- so this + * needs neither {@link java.math.BigDecimal} nor an allocation. {@code Math.round} is not: + * it rounds half up. + * + *

Honest note on how far that matters. The rounded value feeds one comparison, + * {@code nonLatinLetters >= NON_LATIN_MIN_LETTERS}, and the two modes differ only at an exact + * k + 0.5 with k even. At an EVEN threshold both land on the same side of it, so with + * {@code NON_LATIN_MIN_LETTERS} at 10 the mode is unobservable through {@link #analyse} -- + * measured, by a mutant that swapped it and passed the whole corpus. It becomes observable + * the moment the threshold is odd: at 9, a product of 8.5 is 8 half-to-even and 9 half-up. + * Python's rule is matched here so that a change to that table cannot introduce a divergence, + * and the rule itself is pinned by a fixture rather than by the corpus. + */ + static int roundHalfEven(double value) { + return (int) Math.rint(value); + } +} diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java index 4f1b28da4..1cb4339b0 100644 --- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java +++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java @@ -28,8 +28,22 @@ public final class LanguageTables { private LanguageTables() { } - /** One named non-Latin script and the code-point ranges that belong to it. */ + /** + * One named non-Latin script and the code-point ranges that belong to it. + * + *

The array is copied in and out, so a table a caller holds cannot be edited under + * detection. Nothing on the hot path reads it -- {@link #lookupScript} does -- so the copy + * costs nothing that matters. + */ public record Script(String name, int[] ranges) { + public Script { + ranges = ranges.clone(); + } + + @Override + public int[] ranges() { + return ranges.clone(); + } } /** @@ -63,6 +77,97 @@ public record Script(String name, int[] ranges) { new Script("kana", new int[] {12352, 12447, 12448, 12543, 12784, 12799}), new Script("han", new int[] {13312, 19903, 19968, 40959, 63744, 64255})); + /** + * {@link #SCRIPT_RANGES} flattened and sorted by code point, as inclusive + * [lo, hi] pairs, with {@link #SCRIPT_LOOKUP_NAMES} naming each one. + * + *

Detection resolves a script with a binary search over this instead of + * walking the 25 scripts in order the way the reference does. The two agree + * exactly because no two ranges overlap -- the generator refuses to emit this + * table otherwise, since the first-match rule would then be unrepresentable in a + * sorted table -- and a test sweeps every code point to prove it. + */ + private static final int[] SCRIPT_LOOKUP = { + 880, 1023, 1024, 1327, 1328, 1423, 1424, 1535, 1536, 1791, 1872, 1919, 2208, 2303, + 2304, 2431, 2432, 2559, 2560, 2687, 2688, 2815, 2816, 2943, 2944, 3071, 3072, 3199, + 3200, 3327, 3328, 3455, 3456, 3583, 3584, 3711, 3712, 3839, 3840, 4095, 4096, 4255, + 4256, 4351, 4352, 4607, 4608, 4991, 6016, 6143, 7936, 8191, 11744, 11775, 12352, 12447, + 12448, 12543, 12592, 12687, 12784, 12799, 13312, 19903, 19968, 40959, 42560, 42655, + 43232, 43263, 44032, 55215, 63744, 64255, 64336, 65023, 65136, 65279 + }; + + /** + * The script each pair of {@link #SCRIPT_LOOKUP} belongs to, as an index into + * {@link #SCRIPT_NAMES}. + * + *

An index rather than the name, so that counting letters by script needs no + * boxing, no hashing and no string comparison per character: the tally is an + * {@code int[]} and the names are attached once at the end. The reference does a + * dictionary update per non-Latin letter instead. + */ + private static final int[] SCRIPT_LOOKUP_INDEX = { + 0, 1, 2, 3, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 22, 20, 21, 0, + 1, 23, 23, 22, 23, 24, 24, 1, 5, 22, 24, 4, 4 + }; + + /** The named scripts, in the order the reference declares them. */ + private static final String[] SCRIPT_NAMES = { + "greek", "cyrillic", "armenian", "hebrew", "arabic", "devanagari", "bengali", "gurmukhi", + "gujarati", "oriya", "tamil", "telugu", "kannada", "malayalam", "sinhala", "thai", "lao", + "tibetan", "myanmar", "georgian", "ethiopic", "khmer", "hangul", "kana", "han" + }; + + /** How many named scripts there are. */ + public static final int SCRIPT_COUNT = SCRIPT_NAMES.length; + + /** + * The largest code point any named script claims. + * + *

An early exit worth having: every letter above it -- the CJK extension + * planes, the kana supplement, and every astral script -- is counted under + * "other", and that is the text most likely to be long. + */ + public static final int SCRIPT_MAX_CODE_POINT = 0xFEFF; + + /** + * The named script claiming {@code codePoint}, or null when none does. + * + *

Equivalent to scanning {@link #SCRIPT_RANGES} in order and taking the first + * match, in O(log n) comparisons rather than O(n) range tests. + */ + public static String lookupScript(int codePoint) { + int index = lookupScriptIndex(codePoint); + return index < 0 ? null : SCRIPT_NAMES[index]; + } + + /** + * The index into {@link #SCRIPT_NAMES} of the script claiming {@code codePoint}, + * or -1 when none does. + */ + public static int lookupScriptIndex(int codePoint) { + if (codePoint > SCRIPT_MAX_CODE_POINT) { + return -1; + } + int low = 0; + int high = SCRIPT_LOOKUP_INDEX.length - 1; + while (low <= high) { + int mid = (low + high) >>> 1; + if (codePoint < SCRIPT_LOOKUP[mid * 2]) { + high = mid - 1; + } else if (codePoint > SCRIPT_LOOKUP[mid * 2 + 1]) { + low = mid + 1; + } else { + return SCRIPT_LOOKUP_INDEX[mid]; + } + } + return -1; + } + + /** The name of the script at {@code index}. */ + public static String scriptName(int index) { + return SCRIPT_NAMES[index]; + } + /** * Function words per language, in the order Python declares them. * diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java index 2da980c03..5733c9269 100644 --- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java +++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java @@ -5,17 +5,22 @@ // and `--check` fails when the committed copy no longer matches CPython. package com.convaiinnovations.laya.lang; +import java.util.ArrayList; +import java.util.List; + /** * The Unicode properties {@code laya.lang} depends on, recorded from CPython 15.0.0. * - *

Compiled in rather than read from the JDK, because the JDK disagrees with CPython about all - * three of them. {@code Character.isLetter} means the same categories as {@code str.isalpha()}, + *

Compiled in rather than read from the JDK, because the JDK disagrees with CPython about every + * one of them. {@code Character.isLetter} means the same categories as {@code str.isalpha()}, * but JDK 17 ships Unicode 13.0 against CPython's 15.0 and therefore does not know 4863 of the code * points Python calls letters -- it reports them unassigned, which would count text in a * recently-added script under no script at all and route it to the English checkpoint. The * canonical combining class is not exposed by the JDK at all, and the Mn/Mc categories are not a * substitute: they differ from it on 1,460 code points, including U+034F COMBINING GRAPHEME - * JOINER, whose class is zero. + * JOINER, whose class is zero. {@code str.isspace()} has no JDK equivalent either: + * {@code Character.isWhitespace} rejects the non-breaking spaces this set holds, and + * {@code Character.isSpaceChar} rejects tab, newline and every control. * *

A table also makes detection independent of the JDK a consumer runs, which matters more than * it sounds: a JDK upgrade must not silently change which script a request is detected as. @@ -34,113 +39,119 @@ private UnicodeTables() { *

659 ranges covering 136104 code points, recorded * from CPython 15.0.0. */ - private static final int[] ALPHA = { - 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, - 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, - 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, - 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, - 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, - 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, - 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2249, 2308, 2361, - 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448, 2451, 2472, - 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, - 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, - 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, - 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, - 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, - 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, - 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, - 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, - 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, - 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, - 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478, - 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, - 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, - 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, - 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, - 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, - 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, - 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, - 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, - 5761, 5786, 5792, 5866, 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, - 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6264, 6272, 6276, 6279, 6312, - 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601, - 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6988, 7043, 7072, 7086, 7087, - 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7304, 7312, 7354, 7357, 7359, - 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965, - 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, - 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, - 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, - 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, - 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8579, 8580, 11264, 11492, - 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, - 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, - 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, 12293, 12294, - 12337, 12341, 12347, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540, 12543, - 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903, 19968, 42124, - 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, - 42656, 42725, 42775, 42783, 42786, 42888, 42891, 42954, 42960, 42961, 42963, 42963, - 42965, 42969, 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, - 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, - 43360, 43388, 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, - 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43646, 43695, - 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, - 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, - 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44002, 44032, 55203, 55216, 55238, - 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, - 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, - 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, - 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, - 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, - 65599, 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335, - 66349, 66368, 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, - 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66928, 66938, - 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001, - 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461, 67463, 67504, - 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, - 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, - 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096, 68112, 68115, 68117, 68119, - 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68324, 68352, 68405, - 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850, - 68864, 68899, 69248, 69289, 69296, 69297, 69376, 69404, 69415, 69415, 69424, 69445, - 69488, 69505, 69552, 69572, 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749, - 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959, 69968, 70002, - 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, - 70163, 70187, 70207, 70208, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, - 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, - 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493, 70497, 70656, 70708, - 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, 70855, 70855, 71040, 71086, - 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, 71424, 71450, - 71488, 71494, 71680, 71723, 71840, 71903, 71935, 71942, 71945, 71945, 71948, 71955, - 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144, - 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, - 72284, 72329, 72349, 72349, 72368, 72440, 72704, 72712, 72714, 72750, 72768, 72768, - 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, - 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488, - 73490, 73523, 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808, 77824, 78895, - 78913, 78918, 82944, 83526, 92160, 92728, 92736, 92766, 92784, 92862, 92880, 92909, - 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, - 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101589, - 101632, 101640, 110576, 110579, 110581, 110587, 110589, 110590, 110592, 110882, 110898, - 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770, - 113776, 113788, 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966, - 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, - 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, - 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, - 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, - 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, - 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180, 123191, 123197, 123214, - 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124896, 124902, 124904, 124907, - 124909, 124910, 124912, 124926, 124928, 125124, 125184, 125251, 125259, 125259, 126464, - 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, - 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, - 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, - 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, - 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, - 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, - 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101, - 196608, 201546, 201552, 205743 - }; + private static final int[] ALPHA = alphaTable(); + + private static int[] alphaTable() { + return new int[] { + 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, + 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, + 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, + 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, + 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, + 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, + 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2249, + 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448, + 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, + 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, + 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, + 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, + 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, + 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, + 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, + 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, + 3160, 3162, 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, + 3242, 3251, 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340, + 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, + 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, + 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, + 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, + 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, + 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, + 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, + 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, + 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, + 5743, 5759, 5761, 5786, 5792, 5866, 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, + 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6264, 6272, 6276, + 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, + 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6988, 7043, 7072, + 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7304, 7312, 7354, + 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, + 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, + 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, + 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, + 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, + 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8579, 8580, + 11264, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, + 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, + 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, + 12293, 12294, 12337, 12341, 12347, 12348, 12353, 12438, 12445, 12447, 12449, 12538, + 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903, + 19968, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, + 42623, 42653, 42656, 42725, 42775, 42783, 42786, 42888, 42891, 42954, 42960, 42961, + 42963, 42963, 42965, 42969, 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042, + 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, + 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503, + 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, + 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, + 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, + 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44002, 44032, 55203, + 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, + 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, + 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, + 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, + 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, + 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, + 66304, 66335, 66349, 66368, 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, + 66504, 66511, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, + 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, + 66995, 67001, 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461, + 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, + 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, + 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096, 68112, 68115, + 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68324, + 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, + 68800, 68850, 68864, 68899, 69248, 69289, 69296, 69297, 69376, 69404, 69415, 69415, + 69424, 69445, 69488, 69505, 69552, 69572, 69600, 69622, 69635, 69687, 69745, 69746, + 69749, 69749, 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959, + 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, + 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, 70280, 70280, 70282, 70285, + 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416, 70419, 70440, + 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493, 70497, + 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, 70855, 70855, + 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, + 71424, 71450, 71488, 71494, 71680, 71723, 71840, 71903, 71935, 71942, 71945, 71945, + 71948, 71955, 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, + 72106, 72144, 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, + 72272, 72272, 72284, 72329, 72349, 72349, 72368, 72440, 72704, 72712, 72714, 72750, + 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, + 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, + 73476, 73488, 73490, 73523, 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808, + 77824, 78895, 78913, 78918, 82944, 83526, 92160, 92728, 92736, 92766, 92784, 92862, + 92880, 92909, 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93760, 93823, + 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, + 100343, 100352, 101589, 101632, 101640, 110576, 110579, 110581, 110587, 110589, + 110590, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, + 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, + 113817, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, + 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, + 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, + 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, + 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, + 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, + 120779, 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180, 123191, + 123197, 123214, 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124896, + 124902, 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124, 125184, + 125251, 125259, 125259, 126464, 126467, 126469, 126495, 126497, 126498, 126500, + 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, + 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, + 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, + 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, + 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, + 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, + 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 194560, + 195101, 196608, 201546, 201552, 205743 + }; + } /** * Python's {@code [^\\W\\d_]}: letters plus the Nl and No numeric categories, which is @@ -149,121 +160,127 @@ private UnicodeTables() { *

709 ranges covering 137255 code points, recorded * from CPython 15.0.0. */ - private static final int[] WORD = { - 65, 90, 97, 122, 170, 170, 178, 179, 181, 181, 185, 186, 188, 190, 192, 214, 216, 246, - 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, - 895, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, - 1366, 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, - 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, - 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, - 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, - 2249, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, - 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, - 2525, 2527, 2529, 2544, 2545, 2548, 2553, 2556, 2556, 2565, 2570, 2575, 2576, 2579, - 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, - 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, - 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, - 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2935, 2947, - 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, - 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3056, 3058, 3077, 3084, 3086, 3088, 3090, - 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169, 3192, 3198, 3200, - 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3293, - 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, - 3406, 3412, 3414, 3416, 3425, 3440, 3448, 3450, 3455, 3461, 3478, 3482, 3505, 3507, - 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, - 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, - 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3882, 3891, 3904, 3911, 3913, 3948, 3976, - 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, - 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, - 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, - 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, - 4885, 4888, 4954, 4969, 4988, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, - 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, - 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6128, 6137, 6176, 6264, 6272, - 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, - 6571, 6576, 6601, 6618, 6618, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, - 6988, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, - 7304, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, - 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, - 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, - 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8304, 8305, 8308, - 8313, 8319, 8329, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, - 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, - 8521, 8526, 8526, 8528, 8585, 9312, 9371, 9450, 9471, 10102, 10131, 11264, 11492, - 11499, 11502, 11506, 11507, 11517, 11517, 11520, 11557, 11559, 11559, 11565, 11565, - 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, - 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, - 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12445, 12447, - 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12690, 12693, 12704, 12735, - 12784, 12799, 12832, 12841, 12872, 12879, 12881, 12895, 12928, 12937, 12977, 12991, - 13312, 19903, 19968, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, - 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42954, - 42960, 42961, 42963, 42963, 42965, 42969, 42994, 43009, 43011, 43013, 43015, 43018, - 43020, 43042, 43056, 43061, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, - 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, - 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, - 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, - 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, - 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, - 43888, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, - 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, - 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, - 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, - 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, - 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, - 65799, 65843, 65856, 65912, 65930, 65931, 66176, 66204, 66208, 66256, 66273, 66299, - 66304, 66339, 66349, 66378, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, - 66513, 66517, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, - 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, - 66995, 67001, 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461, - 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, - 67644, 67644, 67647, 67669, 67672, 67702, 67705, 67742, 67751, 67759, 67808, 67826, - 67828, 67829, 67835, 67867, 67872, 67897, 67968, 68023, 68028, 68047, 68050, 68096, - 68112, 68115, 68117, 68119, 68121, 68149, 68160, 68168, 68192, 68222, 68224, 68255, - 68288, 68295, 68297, 68324, 68331, 68335, 68352, 68405, 68416, 68437, 68440, 68466, - 68472, 68497, 68521, 68527, 68608, 68680, 68736, 68786, 68800, 68850, 68858, 68899, - 69216, 69246, 69248, 69289, 69296, 69297, 69376, 69415, 69424, 69445, 69457, 69460, - 69488, 69505, 69552, 69579, 69600, 69622, 69635, 69687, 69714, 69733, 69745, 69746, - 69749, 69749, 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959, - 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, - 70113, 70132, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, 70280, 70280, - 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416, - 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, - 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, - 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, - 71352, 71352, 71424, 71450, 71482, 71483, 71488, 71494, 71680, 71723, 71840, 71903, - 71914, 71922, 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71983, - 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163, - 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349, - 72368, 72440, 72704, 72712, 72714, 72750, 72768, 72768, 72794, 72812, 72818, 72847, - 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063, 73064, - 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523, - 73648, 73648, 73664, 73684, 73728, 74649, 74752, 74862, 74880, 75075, 77712, 77808, - 77824, 78895, 78913, 78918, 82944, 83526, 92160, 92728, 92736, 92766, 92784, 92862, - 92880, 92909, 92928, 92975, 92992, 92995, 93019, 93025, 93027, 93047, 93053, 93071, - 93760, 93846, 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, - 94208, 100343, 100352, 101589, 101632, 101640, 110576, 110579, 110581, 110587, 110589, - 110590, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951, - 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119488, - 119507, 119520, 119539, 119648, 119672, 119808, 119892, 119894, 119964, 119966, 119967, - 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, - 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, - 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, - 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, - 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 122624, - 122654, 122661, 122666, 122928, 122989, 123136, 123180, 123191, 123197, 123214, 123214, - 123536, 123565, 123584, 123627, 124112, 124139, 124896, 124902, 124904, 124907, 124909, - 124910, 124912, 124926, 124928, 125124, 125127, 125135, 125184, 125251, 125259, 125259, - 126065, 126123, 126125, 126127, 126129, 126132, 126209, 126253, 126255, 126269, 126464, - 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, - 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, - 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, - 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, - 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, - 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 127232, - 127244, 131072, 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, - 194560, 195101, 196608, 201546, 201552, 205743 - }; + private static final int[] WORD = wordTable(); + + private static int[] wordTable() { + return new int[] { + 65, 90, 97, 122, 170, 170, 178, 179, 181, 181, 185, 186, 188, 190, 192, 214, 216, + 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, + 893, 895, 895, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, + 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, + 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, + 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, + 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, + 2190, 2208, 2249, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, + 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, + 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2548, 2553, 2556, 2556, 2565, 2570, 2575, + 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, + 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, + 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, + 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, + 2935, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, + 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3056, 3058, 3077, 3084, 3086, + 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169, 3192, + 3198, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, + 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386, 3389, + 3389, 3406, 3406, 3412, 3414, 3416, 3425, 3440, 3448, 3450, 3455, 3461, 3478, 3482, + 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, + 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, + 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3882, 3891, 3904, 3911, 3913, + 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, + 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, + 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, + 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, + 4880, 4882, 4885, 4888, 4954, 4969, 4988, 4992, 5007, 5024, 5109, 5112, 5117, 5121, + 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5905, 5919, 5937, 5952, + 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6128, 6137, 6176, + 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, + 6516, 6528, 6571, 6576, 6601, 6618, 6618, 6656, 6678, 6688, 6740, 6823, 6823, 6917, + 6963, 6981, 6988, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, + 7293, 7296, 7304, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, + 7418, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, + 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, + 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8304, + 8305, 8308, 8313, 8319, 8329, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, + 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, + 8511, 8517, 8521, 8526, 8526, 8528, 8585, 9312, 9371, 9450, 9471, 10102, 10131, + 11264, 11492, 11499, 11502, 11506, 11507, 11517, 11517, 11520, 11557, 11559, 11559, + 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, + 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, + 11823, 11823, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, + 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12690, 12693, + 12704, 12735, 12784, 12799, 12832, 12841, 12872, 12879, 12881, 12895, 12928, 12937, + 12977, 12991, 13312, 19903, 19968, 42124, 42192, 42237, 42240, 42508, 42512, 42527, + 42538, 42539, 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, + 42891, 42954, 42960, 42961, 42963, 42963, 42965, 42969, 42994, 43009, 43011, 43013, + 43015, 43018, 43020, 43042, 43056, 43061, 43072, 43123, 43138, 43187, 43250, 43255, + 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, + 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, + 43588, 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, + 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, + 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, + 43868, 43881, 43888, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, + 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, + 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, + 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, + 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, + 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, + 65664, 65786, 65799, 65843, 65856, 65912, 65930, 65931, 66176, 66204, 66208, 66256, + 66273, 66299, 66304, 66339, 66349, 66378, 66384, 66421, 66432, 66461, 66464, 66499, + 66504, 66511, 66513, 66517, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, + 66864, 66915, 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, + 66979, 66993, 66995, 67001, 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, + 67456, 67461, 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, + 67639, 67640, 67644, 67644, 67647, 67669, 67672, 67702, 67705, 67742, 67751, 67759, + 67808, 67826, 67828, 67829, 67835, 67867, 67872, 67897, 67968, 68023, 68028, 68047, + 68050, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68160, 68168, 68192, 68222, + 68224, 68255, 68288, 68295, 68297, 68324, 68331, 68335, 68352, 68405, 68416, 68437, + 68440, 68466, 68472, 68497, 68521, 68527, 68608, 68680, 68736, 68786, 68800, 68850, + 68858, 68899, 69216, 69246, 69248, 69289, 69296, 69297, 69376, 69415, 69424, 69445, + 69457, 69460, 69488, 69505, 69552, 69579, 69600, 69622, 69635, 69687, 69714, 69733, + 69745, 69746, 69749, 69749, 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, + 69959, 69959, 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, + 70108, 70108, 70113, 70132, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, + 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, + 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, + 70480, 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, + 70852, 70853, 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, + 71296, 71338, 71352, 71352, 71424, 71450, 71482, 71483, 71488, 71494, 71680, 71723, + 71840, 71903, 71914, 71922, 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, + 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144, 72161, 72161, + 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, + 72349, 72349, 72368, 72440, 72704, 72712, 72714, 72750, 72768, 72768, 72794, 72812, + 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, + 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488, + 73490, 73523, 73648, 73648, 73664, 73684, 73728, 74649, 74752, 74862, 74880, 75075, + 77712, 77808, 77824, 78895, 78913, 78918, 82944, 83526, 92160, 92728, 92736, 92766, + 92784, 92862, 92880, 92909, 92928, 92975, 92992, 92995, 93019, 93025, 93027, 93047, + 93053, 93071, 93760, 93846, 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177, + 94179, 94179, 94208, 100343, 100352, 101589, 101632, 101640, 110576, 110579, + 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898, 110928, 110930, + 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, + 113792, 113800, 113808, 113817, 119488, 119507, 119520, 119539, 119648, 119672, + 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, + 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, + 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, + 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, + 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, + 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, + 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180, 123191, 123197, + 123214, 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124896, 124902, + 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124, 125127, 125135, + 125184, 125251, 125259, 125259, 126065, 126123, 126125, 126127, 126129, 126132, + 126209, 126253, 126255, 126269, 126464, 126467, 126469, 126495, 126497, 126498, + 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, + 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, + 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, + 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, + 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, + 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, + 127232, 127244, 131072, 173791, 173824, 177977, 177984, 178205, 178208, 183969, + 183984, 191456, 194560, 195101, 196608, 201546, 201552, 205743 + }; + } /** * Code points with a non-zero canonical combining class. @@ -271,38 +288,43 @@ private UnicodeTables() { *

192 ranges covering 922 code points, recorded * from CPython 15.0.0. */ - private static final int[] COMBINING = { - 768, 846, 848, 879, 1155, 1159, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, - 1479, 1552, 1562, 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, - 1773, 1809, 1809, 1840, 1866, 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, 2085, - 2087, 2089, 2093, 2137, 2139, 2200, 2207, 2250, 2273, 2275, 2303, 2364, 2364, 2381, - 2381, 2385, 2388, 2492, 2492, 2509, 2509, 2558, 2558, 2620, 2620, 2637, 2637, 2748, - 2748, 2765, 2765, 2876, 2876, 2893, 2893, 3021, 3021, 3132, 3132, 3149, 3149, 3157, - 3158, 3260, 3260, 3277, 3277, 3387, 3388, 3405, 3405, 3530, 3530, 3640, 3642, 3656, - 3659, 3768, 3770, 3784, 3787, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897, 3953, - 3954, 3956, 3956, 3962, 3965, 3968, 3968, 3970, 3972, 3974, 3975, 4038, 4038, 4151, - 4151, 4153, 4154, 4237, 4237, 4957, 4959, 5908, 5909, 5940, 5940, 6098, 6098, 6109, - 6109, 6313, 6313, 6457, 6459, 6679, 6680, 6752, 6752, 6773, 6780, 6783, 6783, 6832, - 6845, 6847, 6862, 6964, 6964, 6980, 6980, 7019, 7027, 7082, 7083, 7142, 7142, 7154, - 7155, 7223, 7223, 7376, 7378, 7380, 7392, 7394, 7400, 7405, 7405, 7412, 7412, 7416, - 7417, 7616, 7679, 8400, 8412, 8417, 8417, 8421, 8432, 11503, 11505, 11647, 11647, - 11744, 11775, 12330, 12335, 12441, 12442, 42607, 42607, 42612, 42621, 42654, 42655, - 42736, 42737, 43014, 43014, 43052, 43052, 43204, 43204, 43232, 43249, 43307, 43309, - 43347, 43347, 43443, 43443, 43456, 43456, 43696, 43696, 43698, 43700, 43703, 43704, - 43710, 43711, 43713, 43713, 43766, 43766, 44013, 44013, 64286, 64286, 65056, 65071, - 66045, 66045, 66272, 66272, 66422, 66426, 68109, 68109, 68111, 68111, 68152, 68154, - 68159, 68159, 68325, 68326, 68900, 68903, 69291, 69292, 69373, 69375, 69446, 69456, - 69506, 69509, 69702, 69702, 69744, 69744, 69759, 69759, 69817, 69818, 69888, 69890, - 69939, 69940, 70003, 70003, 70080, 70080, 70090, 70090, 70197, 70198, 70377, 70378, - 70459, 70460, 70477, 70477, 70502, 70508, 70512, 70516, 70722, 70722, 70726, 70726, - 70750, 70750, 70850, 70851, 71103, 71104, 71231, 71231, 71350, 71351, 71467, 71467, - 71737, 71738, 71997, 71998, 72003, 72003, 72160, 72160, 72244, 72244, 72263, 72263, - 72345, 72345, 72767, 72767, 73026, 73026, 73028, 73029, 73111, 73111, 73537, 73538, - 92912, 92916, 92976, 92982, 94192, 94193, 113822, 113822, 119141, 119145, 119149, - 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 122880, 122886, - 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123023, 123023, 123184, - 123190, 123566, 123566, 123628, 123631, 124140, 124143, 125136, 125142, 125252, 125258 - }; + private static final int[] COMBINING = combiningTable(); + + private static int[] combiningTable() { + return new int[] { + 768, 846, 848, 879, 1155, 1159, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, + 1479, 1479, 1552, 1562, 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, + 1770, 1773, 1809, 1809, 1840, 1866, 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, + 2085, 2087, 2089, 2093, 2137, 2139, 2200, 2207, 2250, 2273, 2275, 2303, 2364, 2364, + 2381, 2381, 2385, 2388, 2492, 2492, 2509, 2509, 2558, 2558, 2620, 2620, 2637, 2637, + 2748, 2748, 2765, 2765, 2876, 2876, 2893, 2893, 3021, 3021, 3132, 3132, 3149, 3149, + 3157, 3158, 3260, 3260, 3277, 3277, 3387, 3388, 3405, 3405, 3530, 3530, 3640, 3642, + 3656, 3659, 3768, 3770, 3784, 3787, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897, + 3953, 3954, 3956, 3956, 3962, 3965, 3968, 3968, 3970, 3972, 3974, 3975, 4038, 4038, + 4151, 4151, 4153, 4154, 4237, 4237, 4957, 4959, 5908, 5909, 5940, 5940, 6098, 6098, + 6109, 6109, 6313, 6313, 6457, 6459, 6679, 6680, 6752, 6752, 6773, 6780, 6783, 6783, + 6832, 6845, 6847, 6862, 6964, 6964, 6980, 6980, 7019, 7027, 7082, 7083, 7142, 7142, + 7154, 7155, 7223, 7223, 7376, 7378, 7380, 7392, 7394, 7400, 7405, 7405, 7412, 7412, + 7416, 7417, 7616, 7679, 8400, 8412, 8417, 8417, 8421, 8432, 11503, 11505, 11647, + 11647, 11744, 11775, 12330, 12335, 12441, 12442, 42607, 42607, 42612, 42621, 42654, + 42655, 42736, 42737, 43014, 43014, 43052, 43052, 43204, 43204, 43232, 43249, 43307, + 43309, 43347, 43347, 43443, 43443, 43456, 43456, 43696, 43696, 43698, 43700, 43703, + 43704, 43710, 43711, 43713, 43713, 43766, 43766, 44013, 44013, 64286, 64286, 65056, + 65071, 66045, 66045, 66272, 66272, 66422, 66426, 68109, 68109, 68111, 68111, 68152, + 68154, 68159, 68159, 68325, 68326, 68900, 68903, 69291, 69292, 69373, 69375, 69446, + 69456, 69506, 69509, 69702, 69702, 69744, 69744, 69759, 69759, 69817, 69818, 69888, + 69890, 69939, 69940, 70003, 70003, 70080, 70080, 70090, 70090, 70197, 70198, 70377, + 70378, 70459, 70460, 70477, 70477, 70502, 70508, 70512, 70516, 70722, 70722, 70726, + 70726, 70750, 70750, 70850, 70851, 71103, 71104, 71231, 71231, 71350, 71351, 71467, + 71467, 71737, 71738, 71997, 71998, 72003, 72003, 72160, 72160, 72244, 72244, 72263, + 72263, 72345, 72345, 72767, 72767, 73026, 73026, 73028, 73029, 73111, 73111, 73537, + 73538, 92912, 92916, 92976, 92982, 94192, 94193, 113822, 113822, 119141, 119145, + 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, + 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, + 123023, 123023, 123184, 123190, 123566, 123566, 123628, 123631, 124140, 124143, + 125136, 125142, 125252, 125258 + }; + } /** * Python's {@code \\d}: the Nd category. Needed because Python's {@code \\w} is @@ -311,18 +333,23 @@ private UnicodeTables() { *

64 ranges covering 680 code points, recorded * from CPython 15.0.0. */ - private static final int[] DIGIT = { - 48, 57, 1632, 1641, 1776, 1785, 1984, 1993, 2406, 2415, 2534, 2543, 2662, 2671, 2790, - 2799, 2918, 2927, 3046, 3055, 3174, 3183, 3302, 3311, 3430, 3439, 3558, 3567, 3664, - 3673, 3792, 3801, 3872, 3881, 4160, 4169, 4240, 4249, 6112, 6121, 6160, 6169, 6470, - 6479, 6608, 6617, 6784, 6793, 6800, 6809, 6992, 7001, 7088, 7097, 7232, 7241, 7248, - 7257, 42528, 42537, 43216, 43225, 43264, 43273, 43472, 43481, 43504, 43513, 43600, - 43609, 44016, 44025, 65296, 65305, 66720, 66729, 68912, 68921, 69734, 69743, 69872, - 69881, 69942, 69951, 70096, 70105, 70384, 70393, 70736, 70745, 70864, 70873, 71248, - 71257, 71360, 71369, 71472, 71481, 71904, 71913, 72016, 72025, 72784, 72793, 73040, - 73049, 73120, 73129, 73552, 73561, 92768, 92777, 92864, 92873, 93008, 93017, 120782, - 120831, 123200, 123209, 123632, 123641, 124144, 124153, 125264, 125273, 130032, 130041 - }; + private static final int[] DIGIT = digitTable(); + + private static int[] digitTable() { + return new int[] { + 48, 57, 1632, 1641, 1776, 1785, 1984, 1993, 2406, 2415, 2534, 2543, 2662, 2671, + 2790, 2799, 2918, 2927, 3046, 3055, 3174, 3183, 3302, 3311, 3430, 3439, 3558, 3567, + 3664, 3673, 3792, 3801, 3872, 3881, 4160, 4169, 4240, 4249, 6112, 6121, 6160, 6169, + 6470, 6479, 6608, 6617, 6784, 6793, 6800, 6809, 6992, 7001, 7088, 7097, 7232, 7241, + 7248, 7257, 42528, 42537, 43216, 43225, 43264, 43273, 43472, 43481, 43504, 43513, + 43600, 43609, 44016, 44025, 65296, 65305, 66720, 66729, 68912, 68921, 69734, 69743, + 69872, 69881, 69942, 69951, 70096, 70105, 70384, 70393, 70736, 70745, 70864, 70873, + 71248, 71257, 71360, 71369, 71472, 71481, 71904, 71913, 72016, 72025, 72784, 72793, + 73040, 73049, 73120, 73129, 73552, 73561, 92768, 92777, 92864, 92873, 93008, 93017, + 120782, 120831, 123200, 123209, 123632, 123641, 124144, 124153, 125264, 125273, + 130032, 130041 + }; + } /** * Python's {@code str.isupper()} for one character. @@ -330,103 +357,109 @@ private UnicodeTables() { *

651 ranges covering 1951 code points, recorded * from CPython 15.0.0. */ - private static final int[] UPPER = { - 65, 90, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262, 262, 264, 264, 266, 266, - 268, 268, 270, 270, 272, 272, 274, 274, 276, 276, 278, 278, 280, 280, 282, 282, 284, - 284, 286, 286, 288, 288, 290, 290, 292, 292, 294, 294, 296, 296, 298, 298, 300, 300, - 302, 302, 304, 304, 306, 306, 308, 308, 310, 310, 313, 313, 315, 315, 317, 317, 319, - 319, 321, 321, 323, 323, 325, 325, 327, 327, 330, 330, 332, 332, 334, 334, 336, 336, - 338, 338, 340, 340, 342, 342, 344, 344, 346, 346, 348, 348, 350, 350, 352, 352, 354, - 354, 356, 356, 358, 358, 360, 360, 362, 362, 364, 364, 366, 366, 368, 368, 370, 370, - 372, 372, 374, 374, 376, 377, 379, 379, 381, 381, 385, 386, 388, 388, 390, 391, 393, - 395, 398, 401, 403, 404, 406, 408, 412, 413, 415, 416, 418, 418, 420, 420, 422, 423, - 425, 425, 428, 428, 430, 431, 433, 435, 437, 437, 439, 440, 444, 444, 452, 452, 455, - 455, 458, 458, 461, 461, 463, 463, 465, 465, 467, 467, 469, 469, 471, 471, 473, 473, - 475, 475, 478, 478, 480, 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492, - 492, 494, 494, 497, 497, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512, 512, - 514, 514, 516, 516, 518, 518, 520, 520, 522, 522, 524, 524, 526, 526, 528, 528, 530, - 530, 532, 532, 534, 534, 536, 536, 538, 538, 540, 540, 542, 542, 544, 544, 546, 546, - 548, 548, 550, 550, 552, 552, 554, 554, 556, 556, 558, 558, 560, 560, 562, 562, 570, - 571, 573, 574, 577, 577, 579, 582, 584, 584, 586, 586, 588, 588, 590, 590, 880, 880, - 882, 882, 886, 886, 895, 895, 902, 902, 904, 906, 908, 908, 910, 911, 913, 929, 931, - 939, 975, 975, 978, 980, 984, 984, 986, 986, 988, 988, 990, 990, 992, 992, 994, 994, - 996, 996, 998, 998, 1000, 1000, 1002, 1002, 1004, 1004, 1006, 1006, 1012, 1012, 1015, - 1015, 1017, 1018, 1021, 1071, 1120, 1120, 1122, 1122, 1124, 1124, 1126, 1126, 1128, - 1128, 1130, 1130, 1132, 1132, 1134, 1134, 1136, 1136, 1138, 1138, 1140, 1140, 1142, - 1142, 1144, 1144, 1146, 1146, 1148, 1148, 1150, 1150, 1152, 1152, 1162, 1162, 1164, - 1164, 1166, 1166, 1168, 1168, 1170, 1170, 1172, 1172, 1174, 1174, 1176, 1176, 1178, - 1178, 1180, 1180, 1182, 1182, 1184, 1184, 1186, 1186, 1188, 1188, 1190, 1190, 1192, - 1192, 1194, 1194, 1196, 1196, 1198, 1198, 1200, 1200, 1202, 1202, 1204, 1204, 1206, - 1206, 1208, 1208, 1210, 1210, 1212, 1212, 1214, 1214, 1216, 1217, 1219, 1219, 1221, - 1221, 1223, 1223, 1225, 1225, 1227, 1227, 1229, 1229, 1232, 1232, 1234, 1234, 1236, - 1236, 1238, 1238, 1240, 1240, 1242, 1242, 1244, 1244, 1246, 1246, 1248, 1248, 1250, - 1250, 1252, 1252, 1254, 1254, 1256, 1256, 1258, 1258, 1260, 1260, 1262, 1262, 1264, - 1264, 1266, 1266, 1268, 1268, 1270, 1270, 1272, 1272, 1274, 1274, 1276, 1276, 1278, - 1278, 1280, 1280, 1282, 1282, 1284, 1284, 1286, 1286, 1288, 1288, 1290, 1290, 1292, - 1292, 1294, 1294, 1296, 1296, 1298, 1298, 1300, 1300, 1302, 1302, 1304, 1304, 1306, - 1306, 1308, 1308, 1310, 1310, 1312, 1312, 1314, 1314, 1316, 1316, 1318, 1318, 1320, - 1320, 1322, 1322, 1324, 1324, 1326, 1326, 1329, 1366, 4256, 4293, 4295, 4295, 4301, - 4301, 5024, 5109, 7312, 7354, 7357, 7359, 7680, 7680, 7682, 7682, 7684, 7684, 7686, - 7686, 7688, 7688, 7690, 7690, 7692, 7692, 7694, 7694, 7696, 7696, 7698, 7698, 7700, - 7700, 7702, 7702, 7704, 7704, 7706, 7706, 7708, 7708, 7710, 7710, 7712, 7712, 7714, - 7714, 7716, 7716, 7718, 7718, 7720, 7720, 7722, 7722, 7724, 7724, 7726, 7726, 7728, - 7728, 7730, 7730, 7732, 7732, 7734, 7734, 7736, 7736, 7738, 7738, 7740, 7740, 7742, - 7742, 7744, 7744, 7746, 7746, 7748, 7748, 7750, 7750, 7752, 7752, 7754, 7754, 7756, - 7756, 7758, 7758, 7760, 7760, 7762, 7762, 7764, 7764, 7766, 7766, 7768, 7768, 7770, - 7770, 7772, 7772, 7774, 7774, 7776, 7776, 7778, 7778, 7780, 7780, 7782, 7782, 7784, - 7784, 7786, 7786, 7788, 7788, 7790, 7790, 7792, 7792, 7794, 7794, 7796, 7796, 7798, - 7798, 7800, 7800, 7802, 7802, 7804, 7804, 7806, 7806, 7808, 7808, 7810, 7810, 7812, - 7812, 7814, 7814, 7816, 7816, 7818, 7818, 7820, 7820, 7822, 7822, 7824, 7824, 7826, - 7826, 7828, 7828, 7838, 7838, 7840, 7840, 7842, 7842, 7844, 7844, 7846, 7846, 7848, - 7848, 7850, 7850, 7852, 7852, 7854, 7854, 7856, 7856, 7858, 7858, 7860, 7860, 7862, - 7862, 7864, 7864, 7866, 7866, 7868, 7868, 7870, 7870, 7872, 7872, 7874, 7874, 7876, - 7876, 7878, 7878, 7880, 7880, 7882, 7882, 7884, 7884, 7886, 7886, 7888, 7888, 7890, - 7890, 7892, 7892, 7894, 7894, 7896, 7896, 7898, 7898, 7900, 7900, 7902, 7902, 7904, - 7904, 7906, 7906, 7908, 7908, 7910, 7910, 7912, 7912, 7914, 7914, 7916, 7916, 7918, - 7918, 7920, 7920, 7922, 7922, 7924, 7924, 7926, 7926, 7928, 7928, 7930, 7930, 7932, - 7932, 7934, 7934, 7944, 7951, 7960, 7965, 7976, 7983, 7992, 7999, 8008, 8013, 8025, - 8025, 8027, 8027, 8029, 8029, 8031, 8031, 8040, 8047, 8120, 8123, 8136, 8139, 8152, - 8155, 8168, 8172, 8184, 8187, 8450, 8450, 8455, 8455, 8459, 8461, 8464, 8466, 8469, - 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8496, 8499, 8510, - 8511, 8517, 8517, 8544, 8559, 8579, 8579, 9398, 9423, 11264, 11311, 11360, 11360, - 11362, 11364, 11367, 11367, 11369, 11369, 11371, 11371, 11373, 11376, 11378, 11378, - 11381, 11381, 11390, 11392, 11394, 11394, 11396, 11396, 11398, 11398, 11400, 11400, - 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408, 11410, 11410, 11412, 11412, - 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422, 11422, 11424, 11424, - 11426, 11426, 11428, 11428, 11430, 11430, 11432, 11432, 11434, 11434, 11436, 11436, - 11438, 11438, 11440, 11440, 11442, 11442, 11444, 11444, 11446, 11446, 11448, 11448, - 11450, 11450, 11452, 11452, 11454, 11454, 11456, 11456, 11458, 11458, 11460, 11460, - 11462, 11462, 11464, 11464, 11466, 11466, 11468, 11468, 11470, 11470, 11472, 11472, - 11474, 11474, 11476, 11476, 11478, 11478, 11480, 11480, 11482, 11482, 11484, 11484, - 11486, 11486, 11488, 11488, 11490, 11490, 11499, 11499, 11501, 11501, 11506, 11506, - 42560, 42560, 42562, 42562, 42564, 42564, 42566, 42566, 42568, 42568, 42570, 42570, - 42572, 42572, 42574, 42574, 42576, 42576, 42578, 42578, 42580, 42580, 42582, 42582, - 42584, 42584, 42586, 42586, 42588, 42588, 42590, 42590, 42592, 42592, 42594, 42594, - 42596, 42596, 42598, 42598, 42600, 42600, 42602, 42602, 42604, 42604, 42624, 42624, - 42626, 42626, 42628, 42628, 42630, 42630, 42632, 42632, 42634, 42634, 42636, 42636, - 42638, 42638, 42640, 42640, 42642, 42642, 42644, 42644, 42646, 42646, 42648, 42648, - 42650, 42650, 42786, 42786, 42788, 42788, 42790, 42790, 42792, 42792, 42794, 42794, - 42796, 42796, 42798, 42798, 42802, 42802, 42804, 42804, 42806, 42806, 42808, 42808, - 42810, 42810, 42812, 42812, 42814, 42814, 42816, 42816, 42818, 42818, 42820, 42820, - 42822, 42822, 42824, 42824, 42826, 42826, 42828, 42828, 42830, 42830, 42832, 42832, - 42834, 42834, 42836, 42836, 42838, 42838, 42840, 42840, 42842, 42842, 42844, 42844, - 42846, 42846, 42848, 42848, 42850, 42850, 42852, 42852, 42854, 42854, 42856, 42856, - 42858, 42858, 42860, 42860, 42862, 42862, 42873, 42873, 42875, 42875, 42877, 42878, - 42880, 42880, 42882, 42882, 42884, 42884, 42886, 42886, 42891, 42891, 42893, 42893, - 42896, 42896, 42898, 42898, 42902, 42902, 42904, 42904, 42906, 42906, 42908, 42908, - 42910, 42910, 42912, 42912, 42914, 42914, 42916, 42916, 42918, 42918, 42920, 42920, - 42922, 42926, 42928, 42932, 42934, 42934, 42936, 42936, 42938, 42938, 42940, 42940, - 42942, 42942, 42944, 42944, 42946, 42946, 42948, 42951, 42953, 42953, 42960, 42960, - 42966, 42966, 42968, 42968, 42997, 42997, 65313, 65338, 66560, 66599, 66736, 66771, - 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 68736, 68786, 71840, 71871, - 93760, 93791, 119808, 119833, 119860, 119885, 119912, 119937, 119964, 119964, 119966, - 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119989, 120016, 120041, - 120068, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120120, 120121, 120123, - 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120172, 120197, 120224, 120249, - 120276, 120301, 120328, 120353, 120380, 120405, 120432, 120457, 120488, 120512, 120546, - 120570, 120604, 120628, 120662, 120686, 120720, 120744, 120778, 120778, 125184, 125217, - 127280, 127305, 127312, 127337, 127344, 127369 - }; + private static final int[] UPPER = upperTable(); + + private static int[] upperTable() { + return new int[] { + 65, 90, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262, 262, 264, 264, 266, + 266, 268, 268, 270, 270, 272, 272, 274, 274, 276, 276, 278, 278, 280, 280, 282, + 282, 284, 284, 286, 286, 288, 288, 290, 290, 292, 292, 294, 294, 296, 296, 298, + 298, 300, 300, 302, 302, 304, 304, 306, 306, 308, 308, 310, 310, 313, 313, 315, + 315, 317, 317, 319, 319, 321, 321, 323, 323, 325, 325, 327, 327, 330, 330, 332, + 332, 334, 334, 336, 336, 338, 338, 340, 340, 342, 342, 344, 344, 346, 346, 348, + 348, 350, 350, 352, 352, 354, 354, 356, 356, 358, 358, 360, 360, 362, 362, 364, + 364, 366, 366, 368, 368, 370, 370, 372, 372, 374, 374, 376, 377, 379, 379, 381, + 381, 385, 386, 388, 388, 390, 391, 393, 395, 398, 401, 403, 404, 406, 408, 412, + 413, 415, 416, 418, 418, 420, 420, 422, 423, 425, 425, 428, 428, 430, 431, 433, + 435, 437, 437, 439, 440, 444, 444, 452, 452, 455, 455, 458, 458, 461, 461, 463, + 463, 465, 465, 467, 467, 469, 469, 471, 471, 473, 473, 475, 475, 478, 478, 480, + 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492, 492, 494, 494, 497, + 497, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512, 512, 514, 514, 516, + 516, 518, 518, 520, 520, 522, 522, 524, 524, 526, 526, 528, 528, 530, 530, 532, + 532, 534, 534, 536, 536, 538, 538, 540, 540, 542, 542, 544, 544, 546, 546, 548, + 548, 550, 550, 552, 552, 554, 554, 556, 556, 558, 558, 560, 560, 562, 562, 570, + 571, 573, 574, 577, 577, 579, 582, 584, 584, 586, 586, 588, 588, 590, 590, 880, + 880, 882, 882, 886, 886, 895, 895, 902, 902, 904, 906, 908, 908, 910, 911, 913, + 929, 931, 939, 975, 975, 978, 980, 984, 984, 986, 986, 988, 988, 990, 990, 992, + 992, 994, 994, 996, 996, 998, 998, 1000, 1000, 1002, 1002, 1004, 1004, 1006, 1006, + 1012, 1012, 1015, 1015, 1017, 1018, 1021, 1071, 1120, 1120, 1122, 1122, 1124, 1124, + 1126, 1126, 1128, 1128, 1130, 1130, 1132, 1132, 1134, 1134, 1136, 1136, 1138, 1138, + 1140, 1140, 1142, 1142, 1144, 1144, 1146, 1146, 1148, 1148, 1150, 1150, 1152, 1152, + 1162, 1162, 1164, 1164, 1166, 1166, 1168, 1168, 1170, 1170, 1172, 1172, 1174, 1174, + 1176, 1176, 1178, 1178, 1180, 1180, 1182, 1182, 1184, 1184, 1186, 1186, 1188, 1188, + 1190, 1190, 1192, 1192, 1194, 1194, 1196, 1196, 1198, 1198, 1200, 1200, 1202, 1202, + 1204, 1204, 1206, 1206, 1208, 1208, 1210, 1210, 1212, 1212, 1214, 1214, 1216, 1217, + 1219, 1219, 1221, 1221, 1223, 1223, 1225, 1225, 1227, 1227, 1229, 1229, 1232, 1232, + 1234, 1234, 1236, 1236, 1238, 1238, 1240, 1240, 1242, 1242, 1244, 1244, 1246, 1246, + 1248, 1248, 1250, 1250, 1252, 1252, 1254, 1254, 1256, 1256, 1258, 1258, 1260, 1260, + 1262, 1262, 1264, 1264, 1266, 1266, 1268, 1268, 1270, 1270, 1272, 1272, 1274, 1274, + 1276, 1276, 1278, 1278, 1280, 1280, 1282, 1282, 1284, 1284, 1286, 1286, 1288, 1288, + 1290, 1290, 1292, 1292, 1294, 1294, 1296, 1296, 1298, 1298, 1300, 1300, 1302, 1302, + 1304, 1304, 1306, 1306, 1308, 1308, 1310, 1310, 1312, 1312, 1314, 1314, 1316, 1316, + 1318, 1318, 1320, 1320, 1322, 1322, 1324, 1324, 1326, 1326, 1329, 1366, 4256, 4293, + 4295, 4295, 4301, 4301, 5024, 5109, 7312, 7354, 7357, 7359, 7680, 7680, 7682, 7682, + 7684, 7684, 7686, 7686, 7688, 7688, 7690, 7690, 7692, 7692, 7694, 7694, 7696, 7696, + 7698, 7698, 7700, 7700, 7702, 7702, 7704, 7704, 7706, 7706, 7708, 7708, 7710, 7710, + 7712, 7712, 7714, 7714, 7716, 7716, 7718, 7718, 7720, 7720, 7722, 7722, 7724, 7724, + 7726, 7726, 7728, 7728, 7730, 7730, 7732, 7732, 7734, 7734, 7736, 7736, 7738, 7738, + 7740, 7740, 7742, 7742, 7744, 7744, 7746, 7746, 7748, 7748, 7750, 7750, 7752, 7752, + 7754, 7754, 7756, 7756, 7758, 7758, 7760, 7760, 7762, 7762, 7764, 7764, 7766, 7766, + 7768, 7768, 7770, 7770, 7772, 7772, 7774, 7774, 7776, 7776, 7778, 7778, 7780, 7780, + 7782, 7782, 7784, 7784, 7786, 7786, 7788, 7788, 7790, 7790, 7792, 7792, 7794, 7794, + 7796, 7796, 7798, 7798, 7800, 7800, 7802, 7802, 7804, 7804, 7806, 7806, 7808, 7808, + 7810, 7810, 7812, 7812, 7814, 7814, 7816, 7816, 7818, 7818, 7820, 7820, 7822, 7822, + 7824, 7824, 7826, 7826, 7828, 7828, 7838, 7838, 7840, 7840, 7842, 7842, 7844, 7844, + 7846, 7846, 7848, 7848, 7850, 7850, 7852, 7852, 7854, 7854, 7856, 7856, 7858, 7858, + 7860, 7860, 7862, 7862, 7864, 7864, 7866, 7866, 7868, 7868, 7870, 7870, 7872, 7872, + 7874, 7874, 7876, 7876, 7878, 7878, 7880, 7880, 7882, 7882, 7884, 7884, 7886, 7886, + 7888, 7888, 7890, 7890, 7892, 7892, 7894, 7894, 7896, 7896, 7898, 7898, 7900, 7900, + 7902, 7902, 7904, 7904, 7906, 7906, 7908, 7908, 7910, 7910, 7912, 7912, 7914, 7914, + 7916, 7916, 7918, 7918, 7920, 7920, 7922, 7922, 7924, 7924, 7926, 7926, 7928, 7928, + 7930, 7930, 7932, 7932, 7934, 7934, 7944, 7951, 7960, 7965, 7976, 7983, 7992, 7999, + 8008, 8013, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8031, 8040, 8047, 8120, 8123, + 8136, 8139, 8152, 8155, 8168, 8172, 8184, 8187, 8450, 8450, 8455, 8455, 8459, 8461, + 8464, 8466, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, + 8496, 8499, 8510, 8511, 8517, 8517, 8544, 8559, 8579, 8579, 9398, 9423, 11264, + 11311, 11360, 11360, 11362, 11364, 11367, 11367, 11369, 11369, 11371, 11371, 11373, + 11376, 11378, 11378, 11381, 11381, 11390, 11392, 11394, 11394, 11396, 11396, 11398, + 11398, 11400, 11400, 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408, 11410, + 11410, 11412, 11412, 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422, + 11422, 11424, 11424, 11426, 11426, 11428, 11428, 11430, 11430, 11432, 11432, 11434, + 11434, 11436, 11436, 11438, 11438, 11440, 11440, 11442, 11442, 11444, 11444, 11446, + 11446, 11448, 11448, 11450, 11450, 11452, 11452, 11454, 11454, 11456, 11456, 11458, + 11458, 11460, 11460, 11462, 11462, 11464, 11464, 11466, 11466, 11468, 11468, 11470, + 11470, 11472, 11472, 11474, 11474, 11476, 11476, 11478, 11478, 11480, 11480, 11482, + 11482, 11484, 11484, 11486, 11486, 11488, 11488, 11490, 11490, 11499, 11499, 11501, + 11501, 11506, 11506, 42560, 42560, 42562, 42562, 42564, 42564, 42566, 42566, 42568, + 42568, 42570, 42570, 42572, 42572, 42574, 42574, 42576, 42576, 42578, 42578, 42580, + 42580, 42582, 42582, 42584, 42584, 42586, 42586, 42588, 42588, 42590, 42590, 42592, + 42592, 42594, 42594, 42596, 42596, 42598, 42598, 42600, 42600, 42602, 42602, 42604, + 42604, 42624, 42624, 42626, 42626, 42628, 42628, 42630, 42630, 42632, 42632, 42634, + 42634, 42636, 42636, 42638, 42638, 42640, 42640, 42642, 42642, 42644, 42644, 42646, + 42646, 42648, 42648, 42650, 42650, 42786, 42786, 42788, 42788, 42790, 42790, 42792, + 42792, 42794, 42794, 42796, 42796, 42798, 42798, 42802, 42802, 42804, 42804, 42806, + 42806, 42808, 42808, 42810, 42810, 42812, 42812, 42814, 42814, 42816, 42816, 42818, + 42818, 42820, 42820, 42822, 42822, 42824, 42824, 42826, 42826, 42828, 42828, 42830, + 42830, 42832, 42832, 42834, 42834, 42836, 42836, 42838, 42838, 42840, 42840, 42842, + 42842, 42844, 42844, 42846, 42846, 42848, 42848, 42850, 42850, 42852, 42852, 42854, + 42854, 42856, 42856, 42858, 42858, 42860, 42860, 42862, 42862, 42873, 42873, 42875, + 42875, 42877, 42878, 42880, 42880, 42882, 42882, 42884, 42884, 42886, 42886, 42891, + 42891, 42893, 42893, 42896, 42896, 42898, 42898, 42902, 42902, 42904, 42904, 42906, + 42906, 42908, 42908, 42910, 42910, 42912, 42912, 42914, 42914, 42916, 42916, 42918, + 42918, 42920, 42920, 42922, 42926, 42928, 42932, 42934, 42934, 42936, 42936, 42938, + 42938, 42940, 42940, 42942, 42942, 42944, 42944, 42946, 42946, 42948, 42951, 42953, + 42953, 42960, 42960, 42966, 42966, 42968, 42968, 42997, 42997, 65313, 65338, 66560, + 66599, 66736, 66771, 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 68736, + 68786, 71840, 71871, 93760, 93791, 119808, 119833, 119860, 119885, 119912, 119937, + 119964, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, + 119982, 119989, 120016, 120041, 120068, 120069, 120071, 120074, 120077, 120084, + 120086, 120092, 120120, 120121, 120123, 120126, 120128, 120132, 120134, 120134, + 120138, 120144, 120172, 120197, 120224, 120249, 120276, 120301, 120328, 120353, + 120380, 120405, 120432, 120457, 120488, 120512, 120546, 120570, 120604, 120628, + 120662, 120686, 120720, 120744, 120778, 120778, 125184, 125217, 127280, 127305, + 127312, 127337, 127344, 127369 + }; + } /** * Python's {@code str.islower()} for one character. @@ -434,124 +467,433 @@ private UnicodeTables() { *

671 ranges covering 2544 code points, recorded * from CPython 15.0.0. */ - private static final int[] LOWER = { - 97, 122, 170, 170, 181, 181, 186, 186, 223, 246, 248, 255, 257, 257, 259, 259, 261, - 261, 263, 263, 265, 265, 267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277, - 279, 279, 281, 281, 283, 283, 285, 285, 287, 287, 289, 289, 291, 291, 293, 293, 295, - 295, 297, 297, 299, 299, 301, 301, 303, 303, 305, 305, 307, 307, 309, 309, 311, 312, - 314, 314, 316, 316, 318, 318, 320, 320, 322, 322, 324, 324, 326, 326, 328, 329, 331, - 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341, 343, 343, 345, 345, 347, 347, - 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359, 361, 361, 363, 363, 365, - 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 378, 378, 380, 380, 382, 384, - 387, 387, 389, 389, 392, 392, 396, 397, 402, 402, 405, 405, 409, 411, 414, 414, 417, - 417, 419, 419, 421, 421, 424, 424, 426, 427, 429, 429, 432, 432, 436, 436, 438, 438, - 441, 442, 445, 447, 454, 454, 457, 457, 460, 460, 462, 462, 464, 464, 466, 466, 468, - 468, 470, 470, 472, 472, 474, 474, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485, - 487, 487, 489, 489, 491, 491, 493, 493, 495, 496, 499, 499, 501, 501, 505, 505, 507, - 507, 509, 509, 511, 511, 513, 513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523, - 525, 525, 527, 527, 529, 529, 531, 531, 533, 533, 535, 535, 537, 537, 539, 539, 541, - 541, 543, 543, 545, 545, 547, 547, 549, 549, 551, 551, 553, 553, 555, 555, 557, 557, - 559, 559, 561, 561, 563, 569, 572, 572, 575, 576, 578, 578, 583, 583, 585, 585, 587, - 587, 589, 589, 591, 659, 661, 696, 704, 705, 736, 740, 837, 837, 881, 881, 883, 883, - 887, 887, 890, 893, 912, 912, 940, 974, 976, 977, 981, 983, 985, 985, 987, 987, 989, - 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005, - 1005, 1007, 1011, 1013, 1013, 1016, 1016, 1019, 1020, 1072, 1119, 1121, 1121, 1123, - 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137, - 1137, 1139, 1139, 1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151, - 1151, 1153, 1153, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, 1171, 1173, - 1173, 1175, 1175, 1177, 1177, 1179, 1179, 1181, 1181, 1183, 1183, 1185, 1185, 1187, - 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197, 1197, 1199, 1199, 1201, - 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213, 1215, - 1215, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230, - 1231, 1233, 1233, 1235, 1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245, - 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253, 1255, 1255, 1257, 1257, 1259, - 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273, - 1273, 1275, 1275, 1277, 1277, 1279, 1279, 1281, 1281, 1283, 1283, 1285, 1285, 1287, - 1287, 1289, 1289, 1291, 1291, 1293, 1293, 1295, 1295, 1297, 1297, 1299, 1299, 1301, - 1301, 1303, 1303, 1305, 1305, 1307, 1307, 1309, 1309, 1311, 1311, 1313, 1313, 1315, - 1315, 1317, 1317, 1319, 1319, 1321, 1321, 1323, 1323, 1325, 1325, 1327, 1327, 1376, - 1416, 4304, 4346, 4348, 4351, 5112, 5117, 7296, 7304, 7424, 7615, 7681, 7681, 7683, - 7683, 7685, 7685, 7687, 7687, 7689, 7689, 7691, 7691, 7693, 7693, 7695, 7695, 7697, - 7697, 7699, 7699, 7701, 7701, 7703, 7703, 7705, 7705, 7707, 7707, 7709, 7709, 7711, - 7711, 7713, 7713, 7715, 7715, 7717, 7717, 7719, 7719, 7721, 7721, 7723, 7723, 7725, - 7725, 7727, 7727, 7729, 7729, 7731, 7731, 7733, 7733, 7735, 7735, 7737, 7737, 7739, - 7739, 7741, 7741, 7743, 7743, 7745, 7745, 7747, 7747, 7749, 7749, 7751, 7751, 7753, - 7753, 7755, 7755, 7757, 7757, 7759, 7759, 7761, 7761, 7763, 7763, 7765, 7765, 7767, - 7767, 7769, 7769, 7771, 7771, 7773, 7773, 7775, 7775, 7777, 7777, 7779, 7779, 7781, - 7781, 7783, 7783, 7785, 7785, 7787, 7787, 7789, 7789, 7791, 7791, 7793, 7793, 7795, - 7795, 7797, 7797, 7799, 7799, 7801, 7801, 7803, 7803, 7805, 7805, 7807, 7807, 7809, - 7809, 7811, 7811, 7813, 7813, 7815, 7815, 7817, 7817, 7819, 7819, 7821, 7821, 7823, - 7823, 7825, 7825, 7827, 7827, 7829, 7837, 7839, 7839, 7841, 7841, 7843, 7843, 7845, - 7845, 7847, 7847, 7849, 7849, 7851, 7851, 7853, 7853, 7855, 7855, 7857, 7857, 7859, - 7859, 7861, 7861, 7863, 7863, 7865, 7865, 7867, 7867, 7869, 7869, 7871, 7871, 7873, - 7873, 7875, 7875, 7877, 7877, 7879, 7879, 7881, 7881, 7883, 7883, 7885, 7885, 7887, - 7887, 7889, 7889, 7891, 7891, 7893, 7893, 7895, 7895, 7897, 7897, 7899, 7899, 7901, - 7901, 7903, 7903, 7905, 7905, 7907, 7907, 7909, 7909, 7911, 7911, 7913, 7913, 7915, - 7915, 7917, 7917, 7919, 7919, 7921, 7921, 7923, 7923, 7925, 7925, 7927, 7927, 7929, - 7929, 7931, 7931, 7933, 7933, 7935, 7943, 7952, 7957, 7968, 7975, 7984, 7991, 8000, - 8005, 8016, 8023, 8032, 8039, 8048, 8061, 8064, 8071, 8080, 8087, 8096, 8103, 8112, - 8116, 8118, 8119, 8126, 8126, 8130, 8132, 8134, 8135, 8144, 8147, 8150, 8151, 8160, - 8167, 8178, 8180, 8182, 8183, 8305, 8305, 8319, 8319, 8336, 8348, 8458, 8458, 8462, - 8463, 8467, 8467, 8495, 8495, 8500, 8500, 8505, 8505, 8508, 8509, 8518, 8521, 8526, - 8526, 8560, 8575, 8580, 8580, 9424, 9449, 11312, 11359, 11361, 11361, 11365, 11366, - 11368, 11368, 11370, 11370, 11372, 11372, 11377, 11377, 11379, 11380, 11382, 11389, - 11393, 11393, 11395, 11395, 11397, 11397, 11399, 11399, 11401, 11401, 11403, 11403, - 11405, 11405, 11407, 11407, 11409, 11409, 11411, 11411, 11413, 11413, 11415, 11415, - 11417, 11417, 11419, 11419, 11421, 11421, 11423, 11423, 11425, 11425, 11427, 11427, - 11429, 11429, 11431, 11431, 11433, 11433, 11435, 11435, 11437, 11437, 11439, 11439, - 11441, 11441, 11443, 11443, 11445, 11445, 11447, 11447, 11449, 11449, 11451, 11451, - 11453, 11453, 11455, 11455, 11457, 11457, 11459, 11459, 11461, 11461, 11463, 11463, - 11465, 11465, 11467, 11467, 11469, 11469, 11471, 11471, 11473, 11473, 11475, 11475, - 11477, 11477, 11479, 11479, 11481, 11481, 11483, 11483, 11485, 11485, 11487, 11487, - 11489, 11489, 11491, 11492, 11500, 11500, 11502, 11502, 11507, 11507, 11520, 11557, - 11559, 11559, 11565, 11565, 42561, 42561, 42563, 42563, 42565, 42565, 42567, 42567, - 42569, 42569, 42571, 42571, 42573, 42573, 42575, 42575, 42577, 42577, 42579, 42579, - 42581, 42581, 42583, 42583, 42585, 42585, 42587, 42587, 42589, 42589, 42591, 42591, - 42593, 42593, 42595, 42595, 42597, 42597, 42599, 42599, 42601, 42601, 42603, 42603, - 42605, 42605, 42625, 42625, 42627, 42627, 42629, 42629, 42631, 42631, 42633, 42633, - 42635, 42635, 42637, 42637, 42639, 42639, 42641, 42641, 42643, 42643, 42645, 42645, - 42647, 42647, 42649, 42649, 42651, 42653, 42787, 42787, 42789, 42789, 42791, 42791, - 42793, 42793, 42795, 42795, 42797, 42797, 42799, 42801, 42803, 42803, 42805, 42805, - 42807, 42807, 42809, 42809, 42811, 42811, 42813, 42813, 42815, 42815, 42817, 42817, - 42819, 42819, 42821, 42821, 42823, 42823, 42825, 42825, 42827, 42827, 42829, 42829, - 42831, 42831, 42833, 42833, 42835, 42835, 42837, 42837, 42839, 42839, 42841, 42841, - 42843, 42843, 42845, 42845, 42847, 42847, 42849, 42849, 42851, 42851, 42853, 42853, - 42855, 42855, 42857, 42857, 42859, 42859, 42861, 42861, 42863, 42872, 42874, 42874, - 42876, 42876, 42879, 42879, 42881, 42881, 42883, 42883, 42885, 42885, 42887, 42887, - 42892, 42892, 42894, 42894, 42897, 42897, 42899, 42901, 42903, 42903, 42905, 42905, - 42907, 42907, 42909, 42909, 42911, 42911, 42913, 42913, 42915, 42915, 42917, 42917, - 42919, 42919, 42921, 42921, 42927, 42927, 42933, 42933, 42935, 42935, 42937, 42937, - 42939, 42939, 42941, 42941, 42943, 42943, 42945, 42945, 42947, 42947, 42952, 42952, - 42954, 42954, 42961, 42961, 42963, 42963, 42965, 42965, 42967, 42967, 42969, 42969, - 42994, 42996, 42998, 42998, 43000, 43002, 43824, 43866, 43868, 43881, 43888, 43967, - 64256, 64262, 64275, 64279, 65345, 65370, 66600, 66639, 66776, 66811, 66967, 66977, - 66979, 66993, 66995, 67001, 67003, 67004, 67456, 67456, 67459, 67461, 67463, 67504, - 67506, 67514, 68800, 68850, 71872, 71903, 93792, 93823, 119834, 119859, 119886, 119892, - 119894, 119911, 119938, 119963, 119990, 119993, 119995, 119995, 119997, 120003, 120005, - 120015, 120042, 120067, 120094, 120119, 120146, 120171, 120198, 120223, 120250, 120275, - 120302, 120327, 120354, 120379, 120406, 120431, 120458, 120485, 120514, 120538, 120540, - 120545, 120572, 120596, 120598, 120603, 120630, 120654, 120656, 120661, 120688, 120712, - 120714, 120719, 120746, 120770, 120772, 120777, 120779, 120779, 122624, 122633, 122635, - 122654, 122661, 122666, 122928, 122989, 125218, 125251 - }; + private static final int[] LOWER = lowerTable(); + + private static int[] lowerTable() { + return new int[] { + 97, 122, 170, 170, 181, 181, 186, 186, 223, 246, 248, 255, 257, 257, 259, 259, 261, + 261, 263, 263, 265, 265, 267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, + 277, 279, 279, 281, 281, 283, 283, 285, 285, 287, 287, 289, 289, 291, 291, 293, + 293, 295, 295, 297, 297, 299, 299, 301, 301, 303, 303, 305, 305, 307, 307, 309, + 309, 311, 312, 314, 314, 316, 316, 318, 318, 320, 320, 322, 322, 324, 324, 326, + 326, 328, 329, 331, 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341, 343, + 343, 345, 345, 347, 347, 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, + 359, 361, 361, 363, 363, 365, 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, + 375, 378, 378, 380, 380, 382, 384, 387, 387, 389, 389, 392, 392, 396, 397, 402, + 402, 405, 405, 409, 411, 414, 414, 417, 417, 419, 419, 421, 421, 424, 424, 426, + 427, 429, 429, 432, 432, 436, 436, 438, 438, 441, 442, 445, 447, 454, 454, 457, + 457, 460, 460, 462, 462, 464, 464, 466, 466, 468, 468, 470, 470, 472, 472, 474, + 474, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485, 487, 487, 489, 489, 491, + 491, 493, 493, 495, 496, 499, 499, 501, 501, 505, 505, 507, 507, 509, 509, 511, + 511, 513, 513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523, 525, 525, 527, + 527, 529, 529, 531, 531, 533, 533, 535, 535, 537, 537, 539, 539, 541, 541, 543, + 543, 545, 545, 547, 547, 549, 549, 551, 551, 553, 553, 555, 555, 557, 557, 559, + 559, 561, 561, 563, 569, 572, 572, 575, 576, 578, 578, 583, 583, 585, 585, 587, + 587, 589, 589, 591, 659, 661, 696, 704, 705, 736, 740, 837, 837, 881, 881, 883, + 883, 887, 887, 890, 893, 912, 912, 940, 974, 976, 977, 981, 983, 985, 985, 987, + 987, 989, 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, + 1003, 1005, 1005, 1007, 1011, 1013, 1013, 1016, 1016, 1019, 1020, 1072, 1119, 1121, + 1121, 1123, 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, + 1135, 1137, 1137, 1139, 1139, 1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, + 1149, 1151, 1151, 1153, 1153, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, + 1171, 1173, 1173, 1175, 1175, 1177, 1177, 1179, 1179, 1181, 1181, 1183, 1183, 1185, + 1185, 1187, 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197, 1197, 1199, + 1199, 1201, 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, + 1213, 1215, 1215, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, + 1228, 1230, 1231, 1233, 1233, 1235, 1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, + 1243, 1245, 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253, 1255, 1255, 1257, + 1257, 1259, 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, + 1271, 1273, 1273, 1275, 1275, 1277, 1277, 1279, 1279, 1281, 1281, 1283, 1283, 1285, + 1285, 1287, 1287, 1289, 1289, 1291, 1291, 1293, 1293, 1295, 1295, 1297, 1297, 1299, + 1299, 1301, 1301, 1303, 1303, 1305, 1305, 1307, 1307, 1309, 1309, 1311, 1311, 1313, + 1313, 1315, 1315, 1317, 1317, 1319, 1319, 1321, 1321, 1323, 1323, 1325, 1325, 1327, + 1327, 1376, 1416, 4304, 4346, 4348, 4351, 5112, 5117, 7296, 7304, 7424, 7615, 7681, + 7681, 7683, 7683, 7685, 7685, 7687, 7687, 7689, 7689, 7691, 7691, 7693, 7693, 7695, + 7695, 7697, 7697, 7699, 7699, 7701, 7701, 7703, 7703, 7705, 7705, 7707, 7707, 7709, + 7709, 7711, 7711, 7713, 7713, 7715, 7715, 7717, 7717, 7719, 7719, 7721, 7721, 7723, + 7723, 7725, 7725, 7727, 7727, 7729, 7729, 7731, 7731, 7733, 7733, 7735, 7735, 7737, + 7737, 7739, 7739, 7741, 7741, 7743, 7743, 7745, 7745, 7747, 7747, 7749, 7749, 7751, + 7751, 7753, 7753, 7755, 7755, 7757, 7757, 7759, 7759, 7761, 7761, 7763, 7763, 7765, + 7765, 7767, 7767, 7769, 7769, 7771, 7771, 7773, 7773, 7775, 7775, 7777, 7777, 7779, + 7779, 7781, 7781, 7783, 7783, 7785, 7785, 7787, 7787, 7789, 7789, 7791, 7791, 7793, + 7793, 7795, 7795, 7797, 7797, 7799, 7799, 7801, 7801, 7803, 7803, 7805, 7805, 7807, + 7807, 7809, 7809, 7811, 7811, 7813, 7813, 7815, 7815, 7817, 7817, 7819, 7819, 7821, + 7821, 7823, 7823, 7825, 7825, 7827, 7827, 7829, 7837, 7839, 7839, 7841, 7841, 7843, + 7843, 7845, 7845, 7847, 7847, 7849, 7849, 7851, 7851, 7853, 7853, 7855, 7855, 7857, + 7857, 7859, 7859, 7861, 7861, 7863, 7863, 7865, 7865, 7867, 7867, 7869, 7869, 7871, + 7871, 7873, 7873, 7875, 7875, 7877, 7877, 7879, 7879, 7881, 7881, 7883, 7883, 7885, + 7885, 7887, 7887, 7889, 7889, 7891, 7891, 7893, 7893, 7895, 7895, 7897, 7897, 7899, + 7899, 7901, 7901, 7903, 7903, 7905, 7905, 7907, 7907, 7909, 7909, 7911, 7911, 7913, + 7913, 7915, 7915, 7917, 7917, 7919, 7919, 7921, 7921, 7923, 7923, 7925, 7925, 7927, + 7927, 7929, 7929, 7931, 7931, 7933, 7933, 7935, 7943, 7952, 7957, 7968, 7975, 7984, + 7991, 8000, 8005, 8016, 8023, 8032, 8039, 8048, 8061, 8064, 8071, 8080, 8087, 8096, + 8103, 8112, 8116, 8118, 8119, 8126, 8126, 8130, 8132, 8134, 8135, 8144, 8147, 8150, + 8151, 8160, 8167, 8178, 8180, 8182, 8183, 8305, 8305, 8319, 8319, 8336, 8348, 8458, + 8458, 8462, 8463, 8467, 8467, 8495, 8495, 8500, 8500, 8505, 8505, 8508, 8509, 8518, + 8521, 8526, 8526, 8560, 8575, 8580, 8580, 9424, 9449, 11312, 11359, 11361, 11361, + 11365, 11366, 11368, 11368, 11370, 11370, 11372, 11372, 11377, 11377, 11379, 11380, + 11382, 11389, 11393, 11393, 11395, 11395, 11397, 11397, 11399, 11399, 11401, 11401, + 11403, 11403, 11405, 11405, 11407, 11407, 11409, 11409, 11411, 11411, 11413, 11413, + 11415, 11415, 11417, 11417, 11419, 11419, 11421, 11421, 11423, 11423, 11425, 11425, + 11427, 11427, 11429, 11429, 11431, 11431, 11433, 11433, 11435, 11435, 11437, 11437, + 11439, 11439, 11441, 11441, 11443, 11443, 11445, 11445, 11447, 11447, 11449, 11449, + 11451, 11451, 11453, 11453, 11455, 11455, 11457, 11457, 11459, 11459, 11461, 11461, + 11463, 11463, 11465, 11465, 11467, 11467, 11469, 11469, 11471, 11471, 11473, 11473, + 11475, 11475, 11477, 11477, 11479, 11479, 11481, 11481, 11483, 11483, 11485, 11485, + 11487, 11487, 11489, 11489, 11491, 11492, 11500, 11500, 11502, 11502, 11507, 11507, + 11520, 11557, 11559, 11559, 11565, 11565, 42561, 42561, 42563, 42563, 42565, 42565, + 42567, 42567, 42569, 42569, 42571, 42571, 42573, 42573, 42575, 42575, 42577, 42577, + 42579, 42579, 42581, 42581, 42583, 42583, 42585, 42585, 42587, 42587, 42589, 42589, + 42591, 42591, 42593, 42593, 42595, 42595, 42597, 42597, 42599, 42599, 42601, 42601, + 42603, 42603, 42605, 42605, 42625, 42625, 42627, 42627, 42629, 42629, 42631, 42631, + 42633, 42633, 42635, 42635, 42637, 42637, 42639, 42639, 42641, 42641, 42643, 42643, + 42645, 42645, 42647, 42647, 42649, 42649, 42651, 42653, 42787, 42787, 42789, 42789, + 42791, 42791, 42793, 42793, 42795, 42795, 42797, 42797, 42799, 42801, 42803, 42803, + 42805, 42805, 42807, 42807, 42809, 42809, 42811, 42811, 42813, 42813, 42815, 42815, + 42817, 42817, 42819, 42819, 42821, 42821, 42823, 42823, 42825, 42825, 42827, 42827, + 42829, 42829, 42831, 42831, 42833, 42833, 42835, 42835, 42837, 42837, 42839, 42839, + 42841, 42841, 42843, 42843, 42845, 42845, 42847, 42847, 42849, 42849, 42851, 42851, + 42853, 42853, 42855, 42855, 42857, 42857, 42859, 42859, 42861, 42861, 42863, 42872, + 42874, 42874, 42876, 42876, 42879, 42879, 42881, 42881, 42883, 42883, 42885, 42885, + 42887, 42887, 42892, 42892, 42894, 42894, 42897, 42897, 42899, 42901, 42903, 42903, + 42905, 42905, 42907, 42907, 42909, 42909, 42911, 42911, 42913, 42913, 42915, 42915, + 42917, 42917, 42919, 42919, 42921, 42921, 42927, 42927, 42933, 42933, 42935, 42935, + 42937, 42937, 42939, 42939, 42941, 42941, 42943, 42943, 42945, 42945, 42947, 42947, + 42952, 42952, 42954, 42954, 42961, 42961, 42963, 42963, 42965, 42965, 42967, 42967, + 42969, 42969, 42994, 42996, 42998, 42998, 43000, 43002, 43824, 43866, 43868, 43881, + 43888, 43967, 64256, 64262, 64275, 64279, 65345, 65370, 66600, 66639, 66776, 66811, + 66967, 66977, 66979, 66993, 66995, 67001, 67003, 67004, 67456, 67456, 67459, 67461, + 67463, 67504, 67506, 67514, 68800, 68850, 71872, 71903, 93792, 93823, 119834, + 119859, 119886, 119892, 119894, 119911, 119938, 119963, 119990, 119993, 119995, + 119995, 119997, 120003, 120005, 120015, 120042, 120067, 120094, 120119, 120146, + 120171, 120198, 120223, 120250, 120275, 120302, 120327, 120354, 120379, 120406, + 120431, 120458, 120485, 120514, 120538, 120540, 120545, 120572, 120596, 120598, + 120603, 120630, 120654, 120656, 120661, 120688, 120712, 120714, 120719, 120746, + 120770, 120772, 120777, 120779, 120779, 122624, 122633, 122635, 122654, 122661, + 122666, 122928, 122989, 125218, 125251 + }; + } + + /** + * Python's {@code str.isspace()}, which is the set {@code str.strip()} and + * {@code str.split()} separate on. + * + *

10 ranges covering 29 code points, recorded + * from CPython 15.0.0. + */ + private static final int[] SPACE = spaceTable(); + + private static int[] spaceTable() { + return new int[] { + 9, 13, 28, 32, 133, 133, 160, 160, 5760, 5760, 8192, 8202, 8232, 8233, 8239, 8239, + 8287, 8287, 12288, 12288 + }; + } + + /** + * Every code point whose Python lowercase is one DIFFERENT code point, + * sorted. {@link #LOWER_TO} holds what each one maps to. + * + *

1432 entries, recorded from CPython 15.0.0. + */ + private static final int[] LOWER_FROM = lowerFromTable(); + + private static int[] lowerFromTable() { + return new int[] { + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, + 86, 87, 88, 89, 90, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, + 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, + 221, 222, 256, 258, 260, 262, 264, 266, 268, 270, 272, 274, 276, 278, 280, 282, + 284, 286, 288, 290, 292, 294, 296, 298, 300, 302, 306, 308, 310, 313, 315, 317, + 319, 321, 323, 325, 327, 330, 332, 334, 336, 338, 340, 342, 344, 346, 348, 350, + 352, 354, 356, 358, 360, 362, 364, 366, 368, 370, 372, 374, 376, 377, 379, 381, + 385, 386, 388, 390, 391, 393, 394, 395, 398, 399, 400, 401, 403, 404, 406, 407, + 408, 412, 413, 415, 416, 418, 420, 422, 423, 425, 428, 430, 431, 433, 434, 435, + 437, 439, 440, 444, 452, 453, 455, 456, 458, 459, 461, 463, 465, 467, 469, 471, + 473, 475, 478, 480, 482, 484, 486, 488, 490, 492, 494, 497, 498, 500, 502, 503, + 504, 506, 508, 510, 512, 514, 516, 518, 520, 522, 524, 526, 528, 530, 532, 534, + 536, 538, 540, 542, 544, 546, 548, 550, 552, 554, 556, 558, 560, 562, 570, 571, + 573, 574, 577, 579, 580, 581, 582, 584, 586, 588, 590, 880, 882, 886, 895, 902, + 904, 905, 906, 908, 910, 911, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, + 923, 924, 925, 926, 927, 928, 929, 931, 932, 933, 934, 935, 936, 937, 938, 939, + 975, 984, 986, 988, 990, 992, 994, 996, 998, 1000, 1002, 1004, 1006, 1012, 1015, + 1017, 1018, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, + 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, + 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, + 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1120, 1122, 1124, + 1126, 1128, 1130, 1132, 1134, 1136, 1138, 1140, 1142, 1144, 1146, 1148, 1150, 1152, + 1162, 1164, 1166, 1168, 1170, 1172, 1174, 1176, 1178, 1180, 1182, 1184, 1186, 1188, + 1190, 1192, 1194, 1196, 1198, 1200, 1202, 1204, 1206, 1208, 1210, 1212, 1214, 1216, + 1217, 1219, 1221, 1223, 1225, 1227, 1229, 1232, 1234, 1236, 1238, 1240, 1242, 1244, + 1246, 1248, 1250, 1252, 1254, 1256, 1258, 1260, 1262, 1264, 1266, 1268, 1270, 1272, + 1274, 1276, 1278, 1280, 1282, 1284, 1286, 1288, 1290, 1292, 1294, 1296, 1298, 1300, + 1302, 1304, 1306, 1308, 1310, 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, 1329, + 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, + 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, + 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 4256, 4257, 4258, 4259, 4260, + 4261, 4262, 4263, 4264, 4265, 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, + 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282, 4283, 4284, 4285, 4286, 4287, 4288, + 4289, 4290, 4291, 4292, 4293, 4295, 4301, 5024, 5025, 5026, 5027, 5028, 5029, 5030, + 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042, 5043, 5044, + 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057, 5058, + 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, + 5073, 5074, 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, + 5087, 5088, 5089, 5090, 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5100, + 5101, 5102, 5103, 5104, 5105, 5106, 5107, 5108, 5109, 7312, 7313, 7314, 7315, 7316, + 7317, 7318, 7319, 7320, 7321, 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, + 7331, 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7344, + 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7357, 7358, 7359, 7680, + 7682, 7684, 7686, 7688, 7690, 7692, 7694, 7696, 7698, 7700, 7702, 7704, 7706, 7708, + 7710, 7712, 7714, 7716, 7718, 7720, 7722, 7724, 7726, 7728, 7730, 7732, 7734, 7736, + 7738, 7740, 7742, 7744, 7746, 7748, 7750, 7752, 7754, 7756, 7758, 7760, 7762, 7764, + 7766, 7768, 7770, 7772, 7774, 7776, 7778, 7780, 7782, 7784, 7786, 7788, 7790, 7792, + 7794, 7796, 7798, 7800, 7802, 7804, 7806, 7808, 7810, 7812, 7814, 7816, 7818, 7820, + 7822, 7824, 7826, 7828, 7838, 7840, 7842, 7844, 7846, 7848, 7850, 7852, 7854, 7856, + 7858, 7860, 7862, 7864, 7866, 7868, 7870, 7872, 7874, 7876, 7878, 7880, 7882, 7884, + 7886, 7888, 7890, 7892, 7894, 7896, 7898, 7900, 7902, 7904, 7906, 7908, 7910, 7912, + 7914, 7916, 7918, 7920, 7922, 7924, 7926, 7928, 7930, 7932, 7934, 7944, 7945, 7946, + 7947, 7948, 7949, 7950, 7951, 7960, 7961, 7962, 7963, 7964, 7965, 7976, 7977, 7978, + 7979, 7980, 7981, 7982, 7983, 7992, 7993, 7994, 7995, 7996, 7997, 7998, 7999, 8008, + 8009, 8010, 8011, 8012, 8013, 8025, 8027, 8029, 8031, 8040, 8041, 8042, 8043, 8044, + 8045, 8046, 8047, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8088, 8089, 8090, + 8091, 8092, 8093, 8094, 8095, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, 8120, + 8121, 8122, 8123, 8124, 8136, 8137, 8138, 8139, 8140, 8152, 8153, 8154, 8155, 8168, + 8169, 8170, 8171, 8172, 8184, 8185, 8186, 8187, 8188, 8486, 8490, 8491, 8498, 8544, + 8545, 8546, 8547, 8548, 8549, 8550, 8551, 8552, 8553, 8554, 8555, 8556, 8557, 8558, + 8559, 8579, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, + 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, + 11264, 11265, 11266, 11267, 11268, 11269, 11270, 11271, 11272, 11273, 11274, 11275, + 11276, 11277, 11278, 11279, 11280, 11281, 11282, 11283, 11284, 11285, 11286, 11287, + 11288, 11289, 11290, 11291, 11292, 11293, 11294, 11295, 11296, 11297, 11298, 11299, + 11300, 11301, 11302, 11303, 11304, 11305, 11306, 11307, 11308, 11309, 11310, 11311, + 11360, 11362, 11363, 11364, 11367, 11369, 11371, 11373, 11374, 11375, 11376, 11378, + 11381, 11390, 11391, 11392, 11394, 11396, 11398, 11400, 11402, 11404, 11406, 11408, + 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, 11426, 11428, 11430, 11432, + 11434, 11436, 11438, 11440, 11442, 11444, 11446, 11448, 11450, 11452, 11454, 11456, + 11458, 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, 11478, 11480, + 11482, 11484, 11486, 11488, 11490, 11499, 11501, 11506, 42560, 42562, 42564, 42566, + 42568, 42570, 42572, 42574, 42576, 42578, 42580, 42582, 42584, 42586, 42588, 42590, + 42592, 42594, 42596, 42598, 42600, 42602, 42604, 42624, 42626, 42628, 42630, 42632, + 42634, 42636, 42638, 42640, 42642, 42644, 42646, 42648, 42650, 42786, 42788, 42790, + 42792, 42794, 42796, 42798, 42802, 42804, 42806, 42808, 42810, 42812, 42814, 42816, + 42818, 42820, 42822, 42824, 42826, 42828, 42830, 42832, 42834, 42836, 42838, 42840, + 42842, 42844, 42846, 42848, 42850, 42852, 42854, 42856, 42858, 42860, 42862, 42873, + 42875, 42877, 42878, 42880, 42882, 42884, 42886, 42891, 42893, 42896, 42898, 42902, + 42904, 42906, 42908, 42910, 42912, 42914, 42916, 42918, 42920, 42922, 42923, 42924, + 42925, 42926, 42928, 42929, 42930, 42931, 42932, 42934, 42936, 42938, 42940, 42942, + 42944, 42946, 42948, 42949, 42950, 42951, 42953, 42960, 42966, 42968, 42997, 65313, + 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, 65324, 65325, + 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336, 65337, + 65338, 66560, 66561, 66562, 66563, 66564, 66565, 66566, 66567, 66568, 66569, 66570, + 66571, 66572, 66573, 66574, 66575, 66576, 66577, 66578, 66579, 66580, 66581, 66582, + 66583, 66584, 66585, 66586, 66587, 66588, 66589, 66590, 66591, 66592, 66593, 66594, + 66595, 66596, 66597, 66598, 66599, 66736, 66737, 66738, 66739, 66740, 66741, 66742, + 66743, 66744, 66745, 66746, 66747, 66748, 66749, 66750, 66751, 66752, 66753, 66754, + 66755, 66756, 66757, 66758, 66759, 66760, 66761, 66762, 66763, 66764, 66765, 66766, + 66767, 66768, 66769, 66770, 66771, 66928, 66929, 66930, 66931, 66932, 66933, 66934, + 66935, 66936, 66937, 66938, 66940, 66941, 66942, 66943, 66944, 66945, 66946, 66947, + 66948, 66949, 66950, 66951, 66952, 66953, 66954, 66956, 66957, 66958, 66959, 66960, + 66961, 66962, 66964, 66965, 68736, 68737, 68738, 68739, 68740, 68741, 68742, 68743, + 68744, 68745, 68746, 68747, 68748, 68749, 68750, 68751, 68752, 68753, 68754, 68755, + 68756, 68757, 68758, 68759, 68760, 68761, 68762, 68763, 68764, 68765, 68766, 68767, + 68768, 68769, 68770, 68771, 68772, 68773, 68774, 68775, 68776, 68777, 68778, 68779, + 68780, 68781, 68782, 68783, 68784, 68785, 68786, 71840, 71841, 71842, 71843, 71844, + 71845, 71846, 71847, 71848, 71849, 71850, 71851, 71852, 71853, 71854, 71855, 71856, + 71857, 71858, 71859, 71860, 71861, 71862, 71863, 71864, 71865, 71866, 71867, 71868, + 71869, 71870, 71871, 93760, 93761, 93762, 93763, 93764, 93765, 93766, 93767, 93768, + 93769, 93770, 93771, 93772, 93773, 93774, 93775, 93776, 93777, 93778, 93779, 93780, + 93781, 93782, 93783, 93784, 93785, 93786, 93787, 93788, 93789, 93790, 93791, + 125184, 125185, 125186, 125187, 125188, 125189, 125190, 125191, 125192, 125193, + 125194, 125195, 125196, 125197, 125198, 125199, 125200, 125201, 125202, 125203, + 125204, 125205, 125206, 125207, 125208, 125209, 125210, 125211, 125212, 125213, + 125214, 125215, 125216, 125217 + }; + } + + /** + * The lowercase of each entry of {@link #LOWER_FROM}, in the same order. + * + *

1432 entries, recorded from CPython 15.0.0. + */ + private static final int[] LOWER_TO = lowerToTable(); + + private static int[] lowerToTable() { + return new int[] { + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 224, 225, 226, 227, 228, 229, 230, + 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 248, 249, 250, 251, 252, 253, 254, 257, 259, 261, 263, 265, 267, 269, 271, 273, + 275, 277, 279, 281, 283, 285, 287, 289, 291, 293, 295, 297, 299, 301, 303, 307, + 309, 311, 314, 316, 318, 320, 322, 324, 326, 328, 331, 333, 335, 337, 339, 341, + 343, 345, 347, 349, 351, 353, 355, 357, 359, 361, 363, 365, 367, 369, 371, 373, + 375, 255, 378, 380, 382, 595, 387, 389, 596, 392, 598, 599, 396, 477, 601, 603, + 402, 608, 611, 617, 616, 409, 623, 626, 629, 417, 419, 421, 640, 424, 643, 429, + 648, 432, 650, 651, 436, 438, 658, 441, 445, 454, 454, 457, 457, 460, 460, 462, + 464, 466, 468, 470, 472, 474, 476, 479, 481, 483, 485, 487, 489, 491, 493, 495, + 499, 499, 501, 405, 447, 505, 507, 509, 511, 513, 515, 517, 519, 521, 523, 525, + 527, 529, 531, 533, 535, 537, 539, 541, 543, 414, 547, 549, 551, 553, 555, 557, + 559, 561, 563, 11365, 572, 410, 11366, 578, 384, 649, 652, 583, 585, 587, 589, 591, + 881, 883, 887, 1011, 940, 941, 942, 943, 972, 973, 974, 945, 946, 947, 948, 949, + 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 963, 964, 965, 966, + 967, 968, 969, 970, 971, 983, 985, 987, 989, 991, 993, 995, 997, 999, 1001, 1003, + 1005, 1007, 952, 1016, 1010, 1019, 891, 892, 893, 1104, 1105, 1106, 1107, 1108, + 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1072, 1073, 1074, + 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, + 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, + 1103, 1121, 1123, 1125, 1127, 1129, 1131, 1133, 1135, 1137, 1139, 1141, 1143, 1145, + 1147, 1149, 1151, 1153, 1163, 1165, 1167, 1169, 1171, 1173, 1175, 1177, 1179, 1181, + 1183, 1185, 1187, 1189, 1191, 1193, 1195, 1197, 1199, 1201, 1203, 1205, 1207, 1209, + 1211, 1213, 1215, 1231, 1218, 1220, 1222, 1224, 1226, 1228, 1230, 1233, 1235, 1237, + 1239, 1241, 1243, 1245, 1247, 1249, 1251, 1253, 1255, 1257, 1259, 1261, 1263, 1265, + 1267, 1269, 1271, 1273, 1275, 1277, 1279, 1281, 1283, 1285, 1287, 1289, 1291, 1293, + 1295, 1297, 1299, 1301, 1303, 1305, 1307, 1309, 1311, 1313, 1315, 1317, 1319, 1321, + 1323, 1325, 1327, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, + 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, + 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1414, + 11520, 11521, 11522, 11523, 11524, 11525, 11526, 11527, 11528, 11529, 11530, 11531, + 11532, 11533, 11534, 11535, 11536, 11537, 11538, 11539, 11540, 11541, 11542, 11543, + 11544, 11545, 11546, 11547, 11548, 11549, 11550, 11551, 11552, 11553, 11554, 11555, + 11556, 11557, 11559, 11565, 43888, 43889, 43890, 43891, 43892, 43893, 43894, 43895, + 43896, 43897, 43898, 43899, 43900, 43901, 43902, 43903, 43904, 43905, 43906, 43907, + 43908, 43909, 43910, 43911, 43912, 43913, 43914, 43915, 43916, 43917, 43918, 43919, + 43920, 43921, 43922, 43923, 43924, 43925, 43926, 43927, 43928, 43929, 43930, 43931, + 43932, 43933, 43934, 43935, 43936, 43937, 43938, 43939, 43940, 43941, 43942, 43943, + 43944, 43945, 43946, 43947, 43948, 43949, 43950, 43951, 43952, 43953, 43954, 43955, + 43956, 43957, 43958, 43959, 43960, 43961, 43962, 43963, 43964, 43965, 43966, 43967, + 5112, 5113, 5114, 5115, 5116, 5117, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, + 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, + 4326, 4327, 4328, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, + 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4349, 4350, 4351, 7681, 7683, 7685, 7687, + 7689, 7691, 7693, 7695, 7697, 7699, 7701, 7703, 7705, 7707, 7709, 7711, 7713, 7715, + 7717, 7719, 7721, 7723, 7725, 7727, 7729, 7731, 7733, 7735, 7737, 7739, 7741, 7743, + 7745, 7747, 7749, 7751, 7753, 7755, 7757, 7759, 7761, 7763, 7765, 7767, 7769, 7771, + 7773, 7775, 7777, 7779, 7781, 7783, 7785, 7787, 7789, 7791, 7793, 7795, 7797, 7799, + 7801, 7803, 7805, 7807, 7809, 7811, 7813, 7815, 7817, 7819, 7821, 7823, 7825, 7827, + 7829, 223, 7841, 7843, 7845, 7847, 7849, 7851, 7853, 7855, 7857, 7859, 7861, 7863, + 7865, 7867, 7869, 7871, 7873, 7875, 7877, 7879, 7881, 7883, 7885, 7887, 7889, 7891, + 7893, 7895, 7897, 7899, 7901, 7903, 7905, 7907, 7909, 7911, 7913, 7915, 7917, 7919, + 7921, 7923, 7925, 7927, 7929, 7931, 7933, 7935, 7936, 7937, 7938, 7939, 7940, 7941, + 7942, 7943, 7952, 7953, 7954, 7955, 7956, 7957, 7968, 7969, 7970, 7971, 7972, 7973, + 7974, 7975, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 8000, 8001, 8002, 8003, + 8004, 8005, 8017, 8019, 8021, 8023, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, + 8064, 8065, 8066, 8067, 8068, 8069, 8070, 8071, 8080, 8081, 8082, 8083, 8084, 8085, + 8086, 8087, 8096, 8097, 8098, 8099, 8100, 8101, 8102, 8103, 8112, 8113, 8048, 8049, + 8115, 8050, 8051, 8052, 8053, 8131, 8144, 8145, 8054, 8055, 8160, 8161, 8058, 8059, + 8165, 8056, 8057, 8060, 8061, 8179, 969, 107, 229, 8526, 8560, 8561, 8562, 8563, + 8564, 8565, 8566, 8567, 8568, 8569, 8570, 8571, 8572, 8573, 8574, 8575, 8580, 9424, + 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437, 9438, + 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9448, 9449, 11312, 11313, + 11314, 11315, 11316, 11317, 11318, 11319, 11320, 11321, 11322, 11323, 11324, 11325, + 11326, 11327, 11328, 11329, 11330, 11331, 11332, 11333, 11334, 11335, 11336, 11337, + 11338, 11339, 11340, 11341, 11342, 11343, 11344, 11345, 11346, 11347, 11348, 11349, + 11350, 11351, 11352, 11353, 11354, 11355, 11356, 11357, 11358, 11359, 11361, 619, + 7549, 637, 11368, 11370, 11372, 593, 625, 592, 594, 11379, 11382, 575, 576, 11393, + 11395, 11397, 11399, 11401, 11403, 11405, 11407, 11409, 11411, 11413, 11415, 11417, + 11419, 11421, 11423, 11425, 11427, 11429, 11431, 11433, 11435, 11437, 11439, 11441, + 11443, 11445, 11447, 11449, 11451, 11453, 11455, 11457, 11459, 11461, 11463, 11465, + 11467, 11469, 11471, 11473, 11475, 11477, 11479, 11481, 11483, 11485, 11487, 11489, + 11491, 11500, 11502, 11507, 42561, 42563, 42565, 42567, 42569, 42571, 42573, 42575, + 42577, 42579, 42581, 42583, 42585, 42587, 42589, 42591, 42593, 42595, 42597, 42599, + 42601, 42603, 42605, 42625, 42627, 42629, 42631, 42633, 42635, 42637, 42639, 42641, + 42643, 42645, 42647, 42649, 42651, 42787, 42789, 42791, 42793, 42795, 42797, 42799, + 42803, 42805, 42807, 42809, 42811, 42813, 42815, 42817, 42819, 42821, 42823, 42825, + 42827, 42829, 42831, 42833, 42835, 42837, 42839, 42841, 42843, 42845, 42847, 42849, + 42851, 42853, 42855, 42857, 42859, 42861, 42863, 42874, 42876, 7545, 42879, 42881, + 42883, 42885, 42887, 42892, 613, 42897, 42899, 42903, 42905, 42907, 42909, 42911, + 42913, 42915, 42917, 42919, 42921, 614, 604, 609, 620, 618, 670, 647, 669, 43859, + 42933, 42935, 42937, 42939, 42941, 42943, 42945, 42947, 42900, 642, 7566, 42952, + 42954, 42961, 42967, 42969, 42998, 65345, 65346, 65347, 65348, 65349, 65350, 65351, + 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360, 65361, 65362, 65363, + 65364, 65365, 65366, 65367, 65368, 65369, 65370, 66600, 66601, 66602, 66603, 66604, + 66605, 66606, 66607, 66608, 66609, 66610, 66611, 66612, 66613, 66614, 66615, 66616, + 66617, 66618, 66619, 66620, 66621, 66622, 66623, 66624, 66625, 66626, 66627, 66628, + 66629, 66630, 66631, 66632, 66633, 66634, 66635, 66636, 66637, 66638, 66639, 66776, + 66777, 66778, 66779, 66780, 66781, 66782, 66783, 66784, 66785, 66786, 66787, 66788, + 66789, 66790, 66791, 66792, 66793, 66794, 66795, 66796, 66797, 66798, 66799, 66800, + 66801, 66802, 66803, 66804, 66805, 66806, 66807, 66808, 66809, 66810, 66811, 66967, + 66968, 66969, 66970, 66971, 66972, 66973, 66974, 66975, 66976, 66977, 66979, 66980, + 66981, 66982, 66983, 66984, 66985, 66986, 66987, 66988, 66989, 66990, 66991, 66992, + 66993, 66995, 66996, 66997, 66998, 66999, 67000, 67001, 67003, 67004, 68800, 68801, + 68802, 68803, 68804, 68805, 68806, 68807, 68808, 68809, 68810, 68811, 68812, 68813, + 68814, 68815, 68816, 68817, 68818, 68819, 68820, 68821, 68822, 68823, 68824, 68825, + 68826, 68827, 68828, 68829, 68830, 68831, 68832, 68833, 68834, 68835, 68836, 68837, + 68838, 68839, 68840, 68841, 68842, 68843, 68844, 68845, 68846, 68847, 68848, 68849, + 68850, 71872, 71873, 71874, 71875, 71876, 71877, 71878, 71879, 71880, 71881, 71882, + 71883, 71884, 71885, 71886, 71887, 71888, 71889, 71890, 71891, 71892, 71893, 71894, + 71895, 71896, 71897, 71898, 71899, 71900, 71901, 71902, 71903, 93792, 93793, 93794, + 93795, 93796, 93797, 93798, 93799, 93800, 93801, 93802, 93803, 93804, 93805, 93806, + 93807, 93808, 93809, 93810, 93811, 93812, 93813, 93814, 93815, 93816, 93817, 93818, + 93819, 93820, 93821, 93822, 93823, 125218, 125219, 125220, 125221, 125222, 125223, + 125224, 125225, 125226, 125227, 125228, 125229, 125230, 125231, 125232, 125233, + 125234, 125235, 125236, 125237, 125238, 125239, 125240, 125241, 125242, 125243, + 125244, 125245, 125246, 125247, 125248, 125249, 125250, 125251 + }; + } + + /* + * ASCII fast paths. Each pair of masks is computed from the same predicate as + * the range table above, so the fast path cannot disagree with the slow one, and + * both are compared against CPython over every code point by UnicodeTablesTest. + */ + private static final long ALPHA_ASCII_LOW = 0L; + private static final long ALPHA_ASCII_HIGH = 576460743847706622L; + + private static final long WORD_ASCII_LOW = 0L; + private static final long WORD_ASCII_HIGH = 576460743847706622L; + + private static final long COMBINING_ASCII_LOW = 0L; + private static final long COMBINING_ASCII_HIGH = 0L; + + private static final long DIGIT_ASCII_LOW = 287948901175001088L; + private static final long DIGIT_ASCII_HIGH = 0L; + + private static final long UPPER_ASCII_LOW = 0L; + private static final long UPPER_ASCII_HIGH = 134217726L; + + private static final long LOWER_ASCII_LOW = 0L; + private static final long LOWER_ASCII_HIGH = 576460743713488896L; + + private static final long SPACE_ASCII_LOW = 8321515008L; + private static final long SPACE_ASCII_HIGH = 0L; + /** Python's {@code str.isalpha()} for one code point. */ public static boolean isAlpha(int codePoint) { + if (codePoint < 0x80) { + long mask = codePoint < 64 ? ALPHA_ASCII_LOW >>> codePoint + : ALPHA_ASCII_HIGH >>> (codePoint - 64); + return (mask & 1L) != 0L; + } return contains(ALPHA, codePoint); } /** Python's {@code [^\\W\\d_]} for one code point: a word character that is not a digit. */ public static boolean isWordChar(int codePoint) { + if (codePoint < 0x80) { + long mask = codePoint < 64 ? WORD_ASCII_LOW >>> codePoint + : WORD_ASCII_HIGH >>> (codePoint - 64); + return (mask & 1L) != 0L; + } return contains(WORD, codePoint); } /** Whether {@code unicodedata.combining} would return non-zero for this code point. */ public static boolean isCombining(int codePoint) { + if (codePoint < 0x80) { + long mask = codePoint < 64 ? COMBINING_ASCII_LOW >>> codePoint + : COMBINING_ASCII_HIGH >>> (codePoint - 64); + return (mask & 1L) != 0L; + } return contains(COMBINING, codePoint); } /** Python's {@code \d}: the Nd category. The JDK misses 30 of these at Unicode 13. */ public static boolean isDigit(int codePoint) { + if (codePoint < 0x80) { + long mask = codePoint < 64 ? DIGIT_ASCII_LOW >>> codePoint + : DIGIT_ASCII_HIGH >>> (codePoint - 64); + return (mask & 1L) != 0L; + } return contains(DIGIT, codePoint); } @@ -582,11 +924,21 @@ public static boolean isWordOrDigit(int codePoint) { * whose case the JDK does not know would change that decision. */ public static boolean isUpper(int codePoint) { + if (codePoint < 0x80) { + long mask = codePoint < 64 ? UPPER_ASCII_LOW >>> codePoint + : UPPER_ASCII_HIGH >>> (codePoint - 64); + return (mask & 1L) != 0L; + } return contains(UPPER, codePoint); } /** Python's {@code str.islower()} for one code point. */ public static boolean isLower(int codePoint) { + if (codePoint < 0x80) { + long mask = codePoint < 64 ? LOWER_ASCII_LOW >>> codePoint + : LOWER_ASCII_HIGH >>> (codePoint - 64); + return (mask & 1L) != 0L; + } return contains(LOWER, codePoint); } @@ -630,14 +982,97 @@ public static boolean hasLower(String text) { return false; } + /** + * Python's {@code str.isspace()} for one code point. + * + *

A table, because neither JDK predicate is this set. {@code Character.isWhitespace} + * rejects four of its members -- U+0085 NEXT LINE, U+00A0 NO-BREAK SPACE, U+2007 FIGURE SPACE + * and U+202F NARROW NO-BREAK SPACE -- and {@code Character.isSpaceChar} is wrong the other + * way, accepting those while rejecting tab, newline and every control. Detection strips and + * splits on this set, so a no-break space pasted into a ticket would otherwise fuse two words + * into one token that no stopword list holds. + */ + public static boolean isSpace(int codePoint) { + if (codePoint < 0x80) { + long mask = codePoint < 64 ? SPACE_ASCII_LOW >>> codePoint + : SPACE_ASCII_HIGH >>> (codePoint - 64); + return (mask & 1L) != 0L; + } + return contains(SPACE, codePoint); + } + + /** Python's {@code str.strip()} with no argument. */ + public static String strip(String text) { + int start = 0; + int end = text.length(); + while (start < end) { + int codePoint = text.codePointAt(start); + if (!isSpace(codePoint)) { + break; + } + start += Character.charCount(codePoint); + } + while (end > start) { + int codePoint = text.codePointBefore(end); + if (!isSpace(codePoint)) { + break; + } + end -= Character.charCount(codePoint); + } + return text.substring(start, end); + } + + /** Whether {@code text} is empty or entirely whitespace, which is Python's {@code not s.strip()}. */ + public static boolean isBlank(String text) { + int i = 0; + while (i < text.length()) { + int codePoint = text.codePointAt(i); + if (!isSpace(codePoint)) { + return false; + } + i += Character.charCount(codePoint); + } + return true; + } + + /** + * Python's {@code str.split()} with no argument: runs of whitespace separate, and leading or + * trailing whitespace yields no empty token. + */ + public static List splitOnWhitespace(String text) { + List out = new ArrayList<>(); + int i = 0; + int length = text.length(); + while (i < length) { + int codePoint = text.codePointAt(i); + if (isSpace(codePoint)) { + i += Character.charCount(codePoint); + continue; + } + int start = i; + while (i < length) { + int inner = text.codePointAt(i); + if (isSpace(inner)) { + break; + } + i += Character.charCount(inner); + } + out.add(text.substring(start, i)); + } + return out; + } + /** * Python's {@code str.lower()}, which is not Java's. * - *

{@link String#toLowerCase} applies the contextual rules, so {@code "\u0391\u03A3"} - * lowercases to {@code "\u03B1\u03C2"} -- a FINAL sigma -- where Python gives - * {@code "\u03B1\u03C3"}. Lowercasing one code point at a time removes the context and - * therefore the rule. Verified over every code point in Unicode: with the multi-character - * mapping below handled explicitly, the two agree exactly. + *

Two things make this not {@link String#toLowerCase}. It applies the contextual rules, + * so the Greek capital sigma at the end of a word lowercases to a FINAL sigma where Python + * gives an ordinary one -- mapping one code point at a time removes the context and therefore + * the rule. And {@link Character#toLowerCase} follows the JDK's Unicode version: at 13.0 it + * does not know the lowercase of 40 code points CPython 15.0 maps, among them all of + * Vithkuqi, and returns them unchanged. So the mapping is a table here as well, which is what + * makes a routing decision the same on every JDK. A test compares a digest over all + * 1,114,112 code points against one recorded from CPython. */ public static String pythonLower(String text) { StringBuilder out = new StringBuilder(text.length()); @@ -649,12 +1084,39 @@ public static String pythonLower(String text) { if (expanded != null) { out.append(expanded); } else { - out.appendCodePoint(Character.toLowerCase(codePoint)); + out.appendCodePoint(lowerCodePoint(codePoint)); } } return out.toString(); } + /** + * Python's {@code str.lower()} for one code point, from the table rather than from the JDK. + * + *

ASCII is answered without the search, which is most text: CPython lowercases exactly + * A-Z below U+0080, and the digest test over every code point proves this shortcut agrees + * with the table. + */ + public static int lowerCodePoint(int codePoint) { + if (codePoint < 0x80) { + return codePoint >= 'A' && codePoint <= 'Z' ? codePoint + 32 : codePoint; + } + int low = 0; + int high = LOWER_FROM.length - 1; + while (low <= high) { + int mid = (low + high) >>> 1; + int key = LOWER_FROM[mid]; + if (codePoint < key) { + high = mid - 1; + } else if (codePoint > key) { + low = mid + 1; + } else { + return LOWER_TO[mid]; + } + } + return codePoint; + } + /** * The one code point whose Python lowercase is longer than one code point. * diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageDetectionParityTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageDetectionParityTest.java new file mode 100644 index 000000000..b61c61a8b --- /dev/null +++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageDetectionParityTest.java @@ -0,0 +1,440 @@ +package com.convaiinnovations.laya.lang; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.convaiinnovations.laya.Fixtures; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; + +/** + * {@link LanguageDetection} against {@code laya/lang.py}, field for field. + * + *

Every expectation here was recorded by running the reference, never written by hand. That + * matters more for this port than for the others: the reference leans on four regexes, two string + * methods and a rounding rule that Java spells differently, and each difference is silent. A + * hand-written expectation would have encoded what the port does rather than what the reference + * does, which is the failure mode the fixtures exist to prevent. + * + *

Each case asserts the whole output and, for a plain string, every intermediate step too, so a + * failure names the stage that broke -- the identifier substitution, the word scan, the acronym + * blanking, the lowercase mapping -- instead of only reporting a different verdict. + */ +class LanguageDetectionParityTest { + + private static List> cases() { + @SuppressWarnings("unchecked") + List> rows = + (List>) Fixtures.load("lang_detect.json").get("cases"); + return rows; + } + + @SuppressWarnings("unchecked") + private static Map map(Object value) { + return (Map) value; + } + + @SuppressWarnings("unchecked") + private static List list(Object value) { + return (List) value; + } + + private static List strings(Object value) { + List out = new ArrayList<>(); + for (Object item : list(value)) { + out.add((String) item); + } + return out; + } + + /** The state as the reference saw it, rebuilt from the tagged fixture value. */ + private static Object state(Map row) { + Map encoded = map(row.get("state")); + String kind = (String) encoded.get("kind"); + if ("string".equals(kind)) { + return encoded.get("value"); + } + if ("bytes".equals(kind)) { + List values = list(encoded.get("value")); + byte[] out = new byte[values.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = (byte) ((Number) values.get(i)).intValue(); + } + return out; + } + return encoded.get("value"); + } + + private static void assertProfile(String where, Object recorded, Map actual) { + Map expected = map(recorded); + // The ORDER is asserted as well as the contents: the reference builds this mapping with + // Latin first and then every other script in first-encounter order, and callers see it. + // A HashMap here would pass a contents-only check and still be wrong. + assertEquals(new ArrayList<>(expected.keySet()), new ArrayList<>(actual.keySet()), + where + ": script profile keys are in a different order"); + for (Map.Entry entry : expected.entrySet()) { + assertEquals(((Number) entry.getValue()).doubleValue(), actual.get(entry.getKey()), + where + ": share of " + entry.getKey()); + } + } + + @TestFactory + @DisplayName("every recorded case matches the reference in every field") + List parity() { + List tests = new ArrayList<>(); + for (Map row : cases()) { + String name = (String) row.get("name"); + tests.add(DynamicTest.dynamicTest(name, () -> assertCase(name, row))); + } + return tests; + } + + private void assertCase(String name, Map row) { + Object state = state(row); + String text = (String) row.get("state_text"); + + assertEquals(text, LanguageDetection.stateText(state), name + ": stateText"); + assertEquals(row.get("detect_script"), LanguageDetection.detectScript(text), + name + ": detectScript"); + assertProfile(name, row.get("script_profile"), LanguageDetection.scriptProfile(text)); + + Map expectedLatin = map(row.get("latin_profile")); + LanguageDetection.LatinProfile latin = LanguageDetection.latinProfile(text); + assertEquals(expectedLatin.get("language"), latin.language(), + name + ": latinProfile language"); + assertEquals(((Number) expectedLatin.get("english_hits")).intValue(), latin.englishHits(), + name + ": latinProfile englishHits"); + assertEquals(((Number) expectedLatin.get("diacritic_rate")).doubleValue(), + latin.diacriticRate(), name + ": latinProfile diacriticRate"); + assertEquals(expectedLatin.get("looks_non_english"), latin.looksNonEnglish(), + name + ": latinProfile looksNonEnglish"); + assertEquals(latin.language(), LanguageDetection.guessLatinLanguage(text), + name + ": guessLatinLanguage must be latinProfile's language"); + + Map expected = map(row.get("analyse")); + LanguageDetection.Analysis analysis = LanguageDetection.analyse(state); + assertEquals(expected.get("script"), analysis.script(), name + ": analyse script"); + assertProfile(name + " (analyse)", expected.get("script_profile"), + analysis.scriptProfile()); + assertEquals(expected.get("language"), analysis.language(), name + ": analyse language"); + assertEquals(expected.get("is_english"), analysis.english(), name + ": analyse isEnglish"); + assertEquals(expected.get("language_undecided"), analysis.languageUndecided(), + name + ": analyse languageUndecided"); + assertEquals(((Number) expected.get("diacritic_rate")).doubleValue(), + analysis.diacriticRate(), name + ": analyse diacriticRate"); + assertEquals(((Number) expected.get("non_latin_fraction")).doubleValue(), + analysis.nonLatinFraction(), name + ": analyse nonLatinFraction"); + assertEquals(expected.get("mixed_segment"), analysis.mixedSegment(), + name + ": analyse mixedSegment"); + assertEquals(row.get("is_english"), LanguageDetection.isEnglish(state), + name + ": isEnglish"); + + if (row.containsKey("helpers")) { + assertHelpers(name, (String) state, map(row.get("helpers")), latin); + } + } + + private void assertHelpers(String name, String text, Map helpers, + LanguageDetection.LatinProfile latin) { + assertEquals(strings(helpers.get("words")), LanguageDetection.words(text), + name + ": the word scan"); + assertEquals(helpers.get("substitute_identifiers"), + LanguageDetection.substituteIdentifiers(text), + name + ": the identifier substitution"); + assertEquals(strings(helpers.get("non_latin_words")), + LanguageDetection.nonLatinWords(text), name + ": the non-Latin word runs"); + assertEquals(helpers.get("named_prose_language"), + LanguageDetection.namedProseLanguage(text), name + ": namedProseLanguage"); + assertEquals(helpers.get("has_code_line"), LanguageDetection.hasCodeLine(text), + name + ": the code-line scan"); + assertEquals(helpers.get("blank_upper_runs"), LanguageDetection.blankUpperRuns(text), + name + ": the acronym blanking"); + assertEquals(helpers.get("python_lower"), UnicodeTables.pythonLower(text), + name + ": pythonLower"); + assertEquals(helpers.get("strip"), UnicodeTables.strip(text), name + ": strip"); + assertEquals(strings(helpers.get("split_whitespace")), + UnicodeTables.splitOnWhitespace(text), name + ": splitOnWhitespace"); + assertEquals(strings(helpers.get("split_newline")), List.of(text.split("\n", -1)), + name + ": splitting on newline must keep trailing empty fields"); + assertEquals(((Number) helpers.get("count_alpha")).intValue(), + LanguageDetection.countAlpha(text), name + ": the letter count"); + assertEquals(((Number) helpers.get("code_point_length")).intValue(), + text.codePointCount(0, text.length()), name + ": the code-point length"); + + List joined = new ArrayList<>(); + for (String token : UnicodeTables.splitOnWhitespace(text)) { + joined.add(LanguageDetection.hasJoined(token)); + } + List expectedJoined = new ArrayList<>(); + for (Object value : list(helpers.get("has_joined_tokens"))) { + expectedJoined.add((Boolean) value); + } + assertEquals(expectedJoined, joined, name + ": the compound-name scan, token by token"); + + List words = LanguageDetection.words(UnicodeTables.pythonLower( + LanguageDetection.substituteIdentifiers(text).replace("İ", "i"))); + assertEquals(helpers.get("english_rescued_by_words"), + LanguageDetection.englishRescuedByWords(words, latin.diacriticRate()), + name + ": the English rescue"); + } + + @Test + @DisplayName("the corpus covers the branches it was built for") + void coverage() { + // A parity suite that silently stopped exercising a branch would still be green, so the + // corpus itself is asserted. These counts are the reason each group of cases was added; + // if one drops to zero the fixture has lost its point and needs a case back. + int mixedSegments = 0; + int nonEnglish = 0; + int undecided = 0; + int nonLatinScripts = 0; + int namedProse = 0; + int rescued = 0; + int identifiersRemoved = 0; + int acronymsBlanked = 0; + for (Map row : cases()) { + Map analysis = map(row.get("analyse")); + if (analysis.get("mixed_segment") != null) { + mixedSegments++; + } + if (Boolean.FALSE.equals(analysis.get("is_english"))) { + nonEnglish++; + } + if (Boolean.TRUE.equals(analysis.get("language_undecided"))) { + undecided++; + } + String script = (String) analysis.get("script"); + if (!"latin".equals(script) && !"unknown".equals(script)) { + nonLatinScripts++; + } + if (row.containsKey("helpers")) { + Map helpers = map(row.get("helpers")); + String raw = (String) map(row.get("state")).get("value"); + if (helpers.get("named_prose_language") != null) { + namedProse++; + } + if (Boolean.TRUE.equals(helpers.get("english_rescued_by_words"))) { + rescued++; + } + if (!raw.equals(helpers.get("substitute_identifiers"))) { + identifiersRemoved++; + } + if (!raw.equals(helpers.get("blank_upper_runs"))) { + acronymsBlanked++; + } + } + } + assertTrue(cases().size() >= 100, "the corpus shrank to " + cases().size() + " cases"); + assertTrue(mixedSegments >= 5, "only " + mixedSegments + " cases reach the segment scan," + + " which is the branch where a foreign line is found inside English prose"); + assertTrue(nonEnglish >= 40, "only " + nonEnglish + " cases route away from English"); + assertTrue(undecided >= 30, "only " + undecided + " cases are undecided, and undecided is" + + " the answer that is neither English nor a named language"); + assertTrue(nonLatinScripts >= 14, "only " + nonLatinScripts + " non-Latin scripts"); + assertTrue(namedProse >= 15, "only " + namedProse + " cases name a prose language"); + assertTrue(rescued >= 10, "only " + rescued + " cases exercise the English rescue"); + assertTrue(identifiersRemoved >= 5, "only " + identifiersRemoved + " cases hold an" + + " identifier for the substitution to remove"); + assertTrue(acronymsBlanked >= 5, "only " + acronymsBlanked + " cases hold an acronym for" + + " the blanking to remove"); + } + + @Test + @DisplayName("the identifier scan is linear, not quadratic, on a long unbroken token") + void identifierScanIsLinear() { + // The reference's pattern needs a lookbehind to stay linear: without it the greedy run is + // retried at every offset inside a run of word characters and rescans it before failing, + // which cost 205 ms on 4,000 characters against 0.45 ms for ordinary prose. A state is + // user input, so this is a denial-of-service shape and not a tuning question. + // + // Asserted structurally rather than with a timer: 200,000 characters is 4x10^10 steps if + // the scan is quadratic, so a green result here cannot be a fast machine. + String run = "a".repeat(200_000); + assertEquals(run, LanguageDetection.substituteIdentifiers(run), + "a run with no joiner must come back untouched"); + assertEquals(" ", LanguageDetection.substituteIdentifiers(run + "." + run), + "one joiner makes the whole thing a single identifier"); + assertEquals(1, LanguageDetection.words(run).size()); + assertFalse(LanguageDetection.hasJoined(run)); + } + + @Test + @DisplayName("the integer round is half-to-even, pinned against CPython") + void roundToIntIsHalfToEven() { + // Pinned here rather than through the corpus on purpose. The rounded value feeds one + // comparison -- nonLatinLetters >= NON_LATIN_MIN_LETTERS -- and half-to-even and half-up + // differ only at an exact k+0.5 with k even, which at an EVEN threshold both land on the + // same side of. A mutant that swapped the mode therefore passed every recorded case. The + // rule is still matched, so that changing that table cannot introduce a divergence, and + // this is what checks it. + int halfwayCases = 0; + int disagreements = 0; + for (Object entry : list(Fixtures.load("lang_detect.json").get("round_to_int"))) { + List pair = list(entry); + double value = ((Number) pair.get(0)).doubleValue(); + int expected = ((Number) pair.get(1)).intValue(); + assertEquals(expected, LanguageDetection.roundHalfEven(value), + "round(" + value + ") disagrees with CPython"); + if (value == Math.floor(value) + 0.5) { + halfwayCases++; + if ((long) Math.floor(value + 0.5) != expected) { + disagreements++; + } + } + } + assertTrue(halfwayCases >= 20, + "only " + halfwayCases + " halfway values, which are the only ones that can tell" + + " the two modes apart"); + assertTrue(disagreements >= 10, "only " + disagreements + " of the halfway values actually" + + " distinguish half-to-even from half-up, so this test would not notice the" + + " difference"); + } + + @Test + @DisplayName("the mode is unobservable at an even threshold and observable at an odd one") + void theRoundingModeIsUnobservableAtTheCurrentThreshold() { + // The measurement behind the comment on roundHalfEven, kept as a test so the claim is + // checked. If NON_LATIN_MIN_LETTERS ever becomes odd this fails, and the surviving mutant + // stops being explained by a property of the table and starts being a real gap. + int threshold = LanguageTables.NON_LATIN_MIN_LETTERS; + int distinguishing = 0; + for (int k = 0; k <= 4000; k++) { + double halfway = k + 0.5; + boolean even = LanguageDetection.roundHalfEven(halfway) >= threshold; + boolean up = (long) Math.floor(halfway + 0.5) >= threshold; + if (even != up) { + distinguishing++; + } + } + assertEquals(threshold % 2, 0, + "NON_LATIN_MIN_LETTERS is now odd, so the rounding mode IS observable through" + + " analyse and the corpus must grow a case for it"); + assertEquals(0, distinguishing, + "at an even threshold no halfway product can distinguish the two modes, yet " + + distinguishing + " did"); + // ...and it really would be observable one lower, which is what makes the above a fact + // about the threshold rather than about the rounding. + int observableAtNine = 0; + for (int k = 0; k <= 40; k++) { + double halfway = k + 0.5; + if ((LanguageDetection.roundHalfEven(halfway) >= threshold - 1) + != ((long) Math.floor(halfway + 0.5) >= threshold - 1)) { + observableAtNine++; + } + } + assertEquals(1, observableAtNine, + "at a threshold of " + (threshold - 1) + " exactly one halfway product" + + " distinguishes the modes, namely " + (threshold - 2) + ".5"); + } + + @Test + @DisplayName("the two Latin cuts are equivalent only because no script reaches below U+0370") + void theTwoLatinCutsAreInertForNow() { + // scriptOf cuts Latin at U+0250 and the script counter at U+02B0, so they differ over the + // IPA extensions. A mutant that unified them survived -- and this is why: no named script + // range reaches that low, so no code point in the gap can name a script either way. The + // faithful pair of constants is kept, and this test fails if a script range is ever added + // below U+02B0, at which point the difference stops being inert. + int lowest = Integer.MAX_VALUE; + for (LanguageTables.Script script : LanguageTables.SCRIPT_RANGES) { + int[] ranges = script.ranges(); + for (int r = 0; r < ranges.length; r += 2) { + lowest = Math.min(lowest, ranges[r]); + } + } + assertTrue(lowest > 0x02AF, + "a script range now starts at U+" + Integer.toHexString(lowest) + ", inside the" + + " U+0250..U+02AF gap between the two Latin cuts. They are no longer" + + " interchangeable and the corpus needs a case in that range."); + for (int cp = 0x0250; cp <= 0x02AF; cp++) { + assertEquals(null, LanguageDetection.scriptOf(cp), + "U+" + Integer.toHexString(cp) + " names a script, which the gap forbids"); + } + } + + @Test + @DisplayName("no code point whose case the JDK disputes can start a non-Latin run") + void theCaseDisputeCannotReachARunStart() { + // A mutant that read the first letter of a non-Latin run with Character.isUpperCase + // instead of the table survived. The reason is structural: the 40 code points CPython and + // JDK 17 disagree about are all outside every script range, so none of them can be the + // first character of a run. The table is still used -- it is the same predicate + // everywhere else -- and this test fails if a disputed code point ever lands inside a + // range, which would make the mutant a real defect. + int reachable = 0; + StringBuilder found = new StringBuilder(); + for (int cp = 0; cp < 0x110000; cp++) { + if (cp >= 0xD800 && cp <= 0xDFFF) { + continue; + } + if (UnicodeTables.isUpper(cp) == Character.isUpperCase(cp)) { + continue; + } + if (LanguageDetection.scriptOf(cp) != null) { + reachable++; + if (found.length() < 60) { + found.append("U+").append(Integer.toHexString(cp)).append(' '); + } + } + } + assertEquals(0, reachable, + "the JDK disputes the case of " + reachable + " code points that CAN start a" + + " non-Latin run (" + found + "), so the corpus needs a case for one of them"); + } + + @Test + @DisplayName("trailing newlines change nothing, which is why the keep-empties split is inert") + void trailingEmptyLinesAreInert() { + // The reference splits on newline Python-style, keeping trailing empty fields; Java's + // String.split drops them, so the port passes -1. A mutant that dropped the -1 survived, + // and this is the reason: both scans skip any line shorter than seven characters, so an + // empty line can never be selected. The faithful spelling is kept and the equivalence is + // asserted rather than assumed. + String body = "The customer opened this ticket yesterday and we asked for a screenshot.\n" + + "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum"; + LanguageDetection.Analysis plain = LanguageDetection.analyse(body); + for (String suffix : new String[] {"\n", "\n\n", "\n\n\n", "\n \n"}) { + assertEquals(plain, LanguageDetection.analyse(body + suffix), + "a trailing newline changed the verdict"); + } + assertEquals(3, body.concat("\n\n").split("\n", -1).length - 1, + "the split itself does keep the empty fields"); + } + + @Test + @DisplayName("a state nested past the depth limit contributes nothing") + void depthLimit() { + Object deep = "Ich kann mich nicht in mein Konto einloggen und das Passwort"; + for (int i = 0; i < 6; i++) { + deep = List.of(deep); + } + assertEquals("Ich kann mich nicht in mein Konto einloggen und das Passwort", + LanguageDetection.stateText(deep), "six levels of nesting are still read"); + assertEquals("", LanguageDetection.stateText(List.of(deep)), + "the seventh level is not, which is where the reference stops"); + } + + @Test + @DisplayName("an undecodable byte leaf contributes nothing rather than mojibake") + void undecodableBytes() { + byte[] invalid = {(byte) 0xFF, (byte) 0xFE, 'h', 'i'}; + assertEquals("", LanguageDetection.stateText(invalid)); + assertTrue(LanguageDetection.isEnglish(invalid), + "a state with no readable text is left to the English checkpoint"); + } + + @Test + @DisplayName("a Set state is not descended into, because its order is unspecified") + void setStateIsIgnored() { + // Deliberate: Set.of randomises iteration order per JVM run, so reading one would make + // detection non-deterministic. The reference accepts str, bytes, Mapping, list and tuple, + // and a Java Set is none of those. + assertEquals("", LanguageDetection.stateText(java.util.Set.of("a", "b"))); + } +} diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageTablesTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageTablesTest.java new file mode 100644 index 000000000..9a50ed275 --- /dev/null +++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/LanguageTablesTest.java @@ -0,0 +1,244 @@ +package com.convaiinnovations.laya.lang; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.convaiinnovations.laya.Fixtures; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The compiled tables against the fixture recorded from {@code laya/lang.py}. + * + *

{@link LanguageTables} is generated, so this is not a test of code: it is the gate that keeps + * a generated file honest. It is worth having because the tables are derived in Python -- + * {@code _SHARED_WORDS}, {@code _EN_ONLY_WORDS} and {@code _EN_COLLISION_WORDS} are computed at + * import time from the stopword lists -- so a hand-maintained Java copy would drift silently, and + * the drift would show up as a misrouted request rather than as a failure. + * + *

Two orderings get their own tests, because they are data and not formatting: the stopword + * order decides which language wins a tied score, and the script order decides which script claims + * a code point two ranges both cover. + */ +class LanguageTablesTest { + + private static Map tables() { + return Fixtures.load("lang_tables.json"); + } + + @SuppressWarnings("unchecked") + private static List list(Object value) { + return (List) value; + } + + @SuppressWarnings("unchecked") + private static Map map(Object value) { + return (Map) value; + } + + private static Set strings(Object value) { + Set out = new TreeSet<>(); + for (Object item : list(value)) { + out.add((String) item); + } + return out; + } + + private void assertWords(String key, Set compiled) { + assertEquals(strings(tables().get(key)), new TreeSet<>(compiled), + key + " drifted from laya/lang.py; regenerate with" + + " scripts/gen_language_tables.py"); + } + + @Test + @DisplayName("every word list equals the one recorded from Python") + void wordLists() { + assertWords("short_swedish_words", LanguageTables.SHORT_SWEDISH_WORDS); + assertWords("non_en_diacritics", LanguageTables.NON_EN_DIACRITICS); + assertWords("shared_words", LanguageTables.SHARED_WORDS); + assertWords("nordic_overlap_words", LanguageTables.NORDIC_OVERLAP_WORDS); + assertWords("en_only_words", LanguageTables.EN_ONLY_WORDS); + assertWords("en_collision_words", LanguageTables.EN_COLLISION_WORDS); + } + + @Test + @DisplayName("the stopword lists equal Python's, language by language") + void stopWords() { + Map recorded = map(tables().get("stop_words")); + assertEquals(recorded.keySet(), new TreeSet<>(LanguageTables.STOP_WORDS.keySet()), + "a language was added to or dropped from the stopword lists"); + for (Map.Entry entry : recorded.entrySet()) { + assertEquals(strings(entry.getValue()), + new TreeSet<>(LanguageTables.STOP_WORDS.get(entry.getKey())), + "the " + entry.getKey() + " stopword list drifted from laya/lang.py"); + } + } + + @Test + @DisplayName("the stopword ORDER is Python's, because it decides a tied score") + void stopWordOrder() { + List recorded = new ArrayList<>(); + for (Object code : list(tables().get("stop_word_order"))) { + recorded.add((String) code); + } + assertEquals(recorded, new ArrayList<>(LanguageTables.STOP_WORDS.keySet()), + "STOP_WORDS is iterated to score languages and the winner is the FIRST of equal" + + " scores, so this order is data. A Map.of() or a sorted map here would change" + + " which language a tie resolves to."); + } + + @Test + @DisplayName("the script ranges equal Python's, in Python's order") + void scriptRanges() { + List recorded = list(tables().get("script_ranges")); + assertEquals(recorded.size(), LanguageTables.SCRIPT_RANGES.size()); + for (int i = 0; i < recorded.size(); i++) { + Map row = map(recorded.get(i)); + LanguageTables.Script script = LanguageTables.SCRIPT_RANGES.get(i); + assertEquals(row.get("name"), script.name(), "script " + i + " is out of order"); + List ranges = list(row.get("ranges")); + int[] compiled = script.ranges(); + assertEquals(ranges.size() * 2, compiled.length, + script.name() + " has a different number of ranges"); + for (int r = 0; r < ranges.size(); r++) { + List pair = list(ranges.get(r)); + assertEquals(((Number) pair.get(0)).intValue(), compiled[r * 2], + script.name() + " range " + r + " low bound"); + assertEquals(((Number) pair.get(1)).intValue(), compiled[r * 2 + 1], + script.name() + " range " + r + " high bound"); + } + } + } + + @Test + @DisplayName("the thresholds equal Python's, to the bit") + void thresholds() { + Map recorded = map(tables().get("thresholds")); + assertEquals(((Number) recorded.get("NON_EN_DIACRITIC_RATE")).doubleValue(), + LanguageTables.NON_EN_DIACRITIC_RATE); + assertEquals(((Number) recorded.get("ENGLISH_RESCUE_DIACRITIC_RATE")).doubleValue(), + LanguageTables.ENGLISH_RESCUE_DIACRITIC_RATE); + assertEquals(((Number) recorded.get("NON_LATIN_FRACTION")).doubleValue(), + LanguageTables.NON_LATIN_FRACTION); + assertEquals(((Number) recorded.get("NON_LATIN_MIN_FRACTION")).doubleValue(), + LanguageTables.NON_LATIN_MIN_FRACTION); + assertEquals(((Number) recorded.get("NON_LATIN_MIN_LETTERS")).intValue(), + LanguageTables.NON_LATIN_MIN_LETTERS); + } + + @Test + @DisplayName("the flat lookup agrees with the ordered scan on every code point") + void flatLookupMatchesTheOrderedScan() { + // The reference walks the 25 scripts in order and takes the first range that matches. + // Detection does a binary search over one sorted table instead, which is only the same + // function while no two ranges overlap. The generator refuses to emit the table otherwise + // -- and this sweeps all 1,114,112 code points to prove the two have not come apart. + List scripts = LanguageTables.SCRIPT_RANGES; + int[][] ranges = new int[scripts.size()][]; + for (int i = 0; i < scripts.size(); i++) { + ranges[i] = scripts.get(i).ranges(); + } + int disagreements = 0; + int firstDisagreement = -1; + for (int cp = 0; cp < 0x110000; cp++) { + String scanned = null; + outer: + for (int i = 0; i < ranges.length; i++) { + for (int r = 0; r < ranges[i].length; r += 2) { + if (cp >= ranges[i][r] && cp <= ranges[i][r + 1]) { + scanned = scripts.get(i).name(); + break outer; + } + } + } + String searched = LanguageTables.lookupScript(cp); + if (!java.util.Objects.equals(scanned, searched)) { + disagreements++; + if (firstDisagreement < 0) { + firstDisagreement = cp; + } + } + } + assertEquals(0, disagreements, + "the binary search and the ordered scan disagree on " + disagreements + + " code points, first at U+" + + (firstDisagreement < 0 ? "none" : Integer.toHexString(firstDisagreement))); + } + + @Test + @DisplayName("scriptOf over every code point matches the digest recorded from Python") + void scriptOfDigest() { + @SuppressWarnings("unchecked") + Map digests = + (Map) Fixtures.load("lang_detect.json").get("unicode_digests"); + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is required of every JVM", impossible); + } + for (int cp = 0; cp < 0x110000; cp++) { + String name = LanguageDetection.scriptOf(cp); + digest.update(((name == null ? "" : name) + ";").getBytes( + java.nio.charset.StandardCharsets.US_ASCII)); + } + StringBuilder hex = new StringBuilder(64); + for (byte value : digest.digest()) { + hex.append(Character.forDigit((value >> 4) & 0xF, 16)); + hex.append(Character.forDigit(value & 0xF, 16)); + } + assertEquals(digests.get("script_of"), hex.toString(), + "scriptOf disagrees with laya.lang._script_of somewhere in Unicode. Note that its" + + " Latin cut (U+0250) is deliberately LOWER than the one the script counter uses" + + " (U+02B0): the IPA extensions are counted as Latin but name no script."); + } + + @Test + @DisplayName("the early exit above the last named script is where the table says it is") + void scriptMaxCodePoint() { + int highest = -1; + for (LanguageTables.Script script : LanguageTables.SCRIPT_RANGES) { + int[] ranges = script.ranges(); + for (int r = 1; r < ranges.length; r += 2) { + highest = Math.max(highest, ranges[r]); + } + } + assertEquals(highest, LanguageTables.SCRIPT_MAX_CODE_POINT, + "lookupScript skips the search above SCRIPT_MAX_CODE_POINT, so a script range" + + " beyond it would become invisible"); + assertNull(LanguageTables.lookupScript(LanguageTables.SCRIPT_MAX_CODE_POINT + 1)); + assertEquals(-1, LanguageTables.lookupScriptIndex(0x110000 - 1)); + } + + @Test + @DisplayName("a Script hands out copies, so a caller cannot edit the table under detection") + void scriptRangesAreCopied() { + LanguageTables.Script script = LanguageTables.SCRIPT_RANGES.get(0); + int[] first = script.ranges(); + int[] second = script.ranges(); + assertTrue(first != second, "ranges() must not hand out the backing array"); + first[0] = -1; + assertEquals(second[0], script.ranges()[0], "editing the copy changed the table"); + } + + @Test + @DisplayName("SCRIPT_COUNT is the number of named scripts, which sizes the tally") + void scriptCount() { + assertEquals(LanguageTables.SCRIPT_RANGES.size(), LanguageTables.SCRIPT_COUNT); + assertSame(LanguageTables.SCRIPT_RANGES.get(0).name(), + LanguageTables.scriptName(0), + "scriptName must index the same declaration order the tally slots use"); + assertThrows(ArrayIndexOutOfBoundsException.class, + () -> LanguageTables.scriptName(LanguageTables.SCRIPT_COUNT)); + } +} diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.java new file mode 100644 index 000000000..d07132a83 --- /dev/null +++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.java @@ -0,0 +1,204 @@ +package com.convaiinnovations.laya.lang; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.convaiinnovations.laya.Fixtures; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; +import java.util.function.IntPredicate; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The character tables, against CPython, over every code point there is. + * + *

A digest per property rather than a sampled corpus. These tables decide which script a + * request is counted under, so "agrees on the cases we thought of" is the wrong assurance: each + * test below compares a sha256 over all 1,114,112 code points against one recorded from CPython, + * and fails the moment a table edit, a CPython bump or a JDK upgrade moves a single one of them. + * + *

Characters are built with {@link #ch} rather than written as unicode escapes. A + * backslash-u escape in Java source is translated before the file is even lexed, so an escape for + * a control or format character is a hazard in a source file rather than a literal in a string. + */ +class UnicodeTablesTest { + + private static final int MAX = 0x110000; + + /** One code point as a string, without putting a unicode escape in this source file. */ + private static String ch(int codePoint) { + return new String(Character.toChars(codePoint)); + } + + @SuppressWarnings("unchecked") + private static Map digests() { + return (Map) Fixtures.load("lang_detect.json").get("unicode_digests"); + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is required of every JVM", impossible); + } + } + + private static String hex(MessageDigest digest) { + StringBuilder out = new StringBuilder(64); + for (byte value : digest.digest()) { + out.append(Character.forDigit((value >> 4) & 0xF, 16)); + out.append(Character.forDigit(value & 0xF, 16)); + } + return out.toString(); + } + + /** The digest Python records for a boolean property: one byte per code point, in order. */ + private static String predicateDigest(IntPredicate predicate) { + MessageDigest digest = sha256(); + byte[] one = {'1'}; + byte[] zero = {'0'}; + for (int cp = 0; cp < MAX; cp++) { + digest.update(predicate.test(cp) ? one : zero); + } + return hex(digest); + } + + private void assertPredicate(String name, IntPredicate predicate) { + assertEquals(digests().get(name), predicateDigest(predicate), + name + ": the compiled table no longer agrees with CPython over all of Unicode." + + " Regenerate with scripts/gen_unicode_tables.py and read the diff before" + + " committing it -- a moved code point changes a routing decision."); + } + + @Test + @DisplayName("the recorded Unicode version is the one the tables were built from") + void version() { + assertEquals(Fixtures.load("lang_detect.json").get("unicode_version"), + UnicodeTables.UNICODE_VERSION); + } + + @Test + @DisplayName("isAlpha is Python's str.isalpha on every code point") + void alpha() { + assertPredicate("alpha", UnicodeTables::isAlpha); + } + + @Test + @DisplayName("isWordChar is Python's word-not-digit class on every code point") + void word() { + assertPredicate("word", UnicodeTables::isWordChar); + } + + @Test + @DisplayName("isCombining is a non-zero canonical combining class on every code point") + void combining() { + assertPredicate("combining", UnicodeTables::isCombining); + } + + @Test + @DisplayName("isDigit is Python's digit class on every code point") + void digit() { + assertPredicate("digit", UnicodeTables::isDigit); + } + + @Test + @DisplayName("isUpper is Python's str.isupper on every code point") + void upper() { + assertPredicate("upper", UnicodeTables::isUpper); + } + + @Test + @DisplayName("isLower is Python's str.islower on every code point") + void lower() { + assertPredicate("lower", UnicodeTables::isLower); + } + + @Test + @DisplayName("isSpace is Python's str.isspace on every code point") + void space() { + assertPredicate("space", UnicodeTables::isSpace); + } + + @Test + @DisplayName("pythonLower maps every code point the way Python's str.lower does") + void pythonLower() { + MessageDigest digest = sha256(); + StringBuilder row = new StringBuilder(32); + for (int cp = 0; cp < MAX; cp++) { + if (cp >= 0xD800 && cp <= 0xDFFF) { + continue; // a lone surrogate is not a character a String can hold + } + String mapped = UnicodeTables.pythonLower(ch(cp)); + row.setLength(0); + row.append(cp).append(':'); + int i = 0; + boolean first = true; + while (i < mapped.length()) { + int lowered = mapped.codePointAt(i); + i += Character.charCount(lowered); + if (!first) { + row.append(','); + } + row.append(lowered); + first = false; + } + row.append(';'); + digest.update(row.toString().getBytes(StandardCharsets.US_ASCII)); + } + assertEquals(digests().get("python_lower"), hex(digest), + "pythonLower no longer agrees with CPython's str.lower. String.toLowerCase is not" + + " a substitute: it applies the contextual final-sigma rule."); + } + + @Test + @DisplayName("the JDK's own predicates would NOT pass, which is why these tables exist") + void theJdkDisagrees() { + // Not a tautology: this is the measurement behind the comment on UnicodeTables, kept as a + // test so the claim is checked rather than asserted. If a future JDK ever catches up, this + // fails and the table can be reconsidered deliberately rather than kept out of habit. + assertFalse(predicateDigest(Character::isLetter).equals(digests().get("alpha")), + "Character.isLetter now agrees with CPython; the ALPHA table may be removable"); + assertFalse(predicateDigest(Character::isWhitespace).equals(digests().get("space")), + "Character.isWhitespace now agrees with CPython's isspace"); + assertFalse(predicateDigest(Character::isUpperCase).equals(digests().get("upper")), + "Character.isUpperCase now agrees with CPython's isupper"); + assertFalse(predicateDigest(cp -> { + int type = Character.getType(cp); + return type == Character.NON_SPACING_MARK || type == Character.COMBINING_SPACING_MARK; + }).equals(digests().get("combining")), + "the Mn/Mc approximation -- which the .NET port uses -- now agrees with the" + + " canonical combining class"); + } + + @Test + @DisplayName("strip and splitOnWhitespace follow isSpace, not Character.isWhitespace") + void stripAndSplit() { + // U+00A0 is whitespace to Python and not to the JDK, which is the whole point: a ticket + // pasted out of a word processor is full of them, and splitting on the JDK's set fuses + // two words into one token that no stopword list holds. + String nbsp = ch(0x00A0); + String nel = ch(0x0085); + String narrowNbsp = ch(0x202F); + String figureSpace = ch(0x2007); + assertEquals("a" + nbsp + "b", + UnicodeTables.strip(nel + " a" + nbsp + "b " + narrowNbsp)); + assertEquals(List.of("a", "b"), + UnicodeTables.splitOnWhitespace(" a" + nbsp + nbsp + "b ")); + assertEquals(List.of(), UnicodeTables.splitOnWhitespace(" " + figureSpace + nel + " ")); + assertTrue(UnicodeTables.isBlank(nbsp + figureSpace + narrowNbsp + nel)); + assertFalse(UnicodeTables.isBlank(ch(0x200B))); // zero-width space is NOT isspace + assertEquals("", UnicodeTables.strip(nbsp)); + // and the JDK really does disagree about each of those four + for (int cp : new int[] {0x0085, 0x00A0, 0x2007, 0x202F}) { + assertTrue(UnicodeTables.isSpace(cp), "Python calls U+" + Integer.toHexString(cp) + + " whitespace"); + assertFalse(Character.isWhitespace(cp), "the JDK does not, which is the reason for" + + " the SPACE table"); + } + } +} diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.py index 422d58cc3..dd7bd5fce 100644 --- a/laya-java/scripts/gen_fixtures.py +++ b/laya-java/scripts/gen_fixtures.py @@ -20,7 +20,9 @@ import argparse import json import os +import re import sys +import unicodedata HERE = os.path.dirname(os.path.abspath(__file__)) FIXTURES = os.path.normpath(os.path.join(HERE, "..", "fixtures")) @@ -50,6 +52,11 @@ def lang_tables(): {"name": name, "ranges": [[int(lo), int(hi)] for lo, hi in ranges]} for name, ranges in lang._SCRIPT_RANGES ], + # Sorted so the committed file diffs stably -- but sorting LOSES the declaration order, + # and that order is data: the language that wins a tied score is the first one iterated. + # So it is recorded separately rather than inferred from this mapping, which is exactly + # the mistake `render`'s missing `sort_keys` was already fixed for once. + "stop_word_order": list(lang._STOP.keys()), "stop_words": {code: _sorted_set(words) for code, words in sorted(lang._STOP.items())}, "short_swedish_words": _sorted_set(lang._SHORT_SWEDISH_WORDS), "non_en_diacritics": _sorted_set(lang._NON_EN_DIACRITICS), @@ -73,6 +80,355 @@ def lang_tables(): } +def _state_repr(state): + """A state as JSON, with `bytes` tagged so a port can rebuild the exact leaf.""" + if isinstance(state, (bytes, bytearray)): + return {"kind": "bytes", "value": list(state)} + if isinstance(state, str): + return {"kind": "string", "value": state} + return {"kind": "json", "value": state} + + +def _unicode_digests(): + """One sha256 per character property, over every code point. + + A digest rather than a table: the committed Java tables already hold the ranges, and what + needs proving is that they still agree with CPython over the WHOLE of Unicode and not just + over the corpus below. Nine hashes do that in 500 bytes of fixture, and they fail the moment + a JDK upgrade, a table edit or a CPython bump moves a single code point. + + Surrogates are included for the predicates -- Python reports every one of them false, and so + must a port -- and excluded from the lowercase digest, which no port can represent. + """ + import hashlib + + word = re.compile(r"[^\W\d_]", re.UNICODE) + digit = re.compile(r"\d", re.UNICODE) + predicates = { + "alpha": lambda ch: ch.isalpha(), + "word": lambda ch: word.fullmatch(ch) is not None, + "combining": lambda ch: unicodedata.combining(ch) != 0, + "digit": lambda ch: digit.fullmatch(ch) is not None, + "upper": lambda ch: ch.isupper(), + "lower": lambda ch: ch.islower(), + "space": lambda ch: ch.isspace(), + } + out = {} + for name, predicate in predicates.items(): + digest = hashlib.sha256() + for cp in range(0x110000): + digest.update(b"1" if predicate(chr(cp)) else b"0") + out[name] = digest.hexdigest() + + lower = hashlib.sha256() + for cp in range(0x110000): + if 0xD800 <= cp <= 0xDFFF: + continue + mapped = ",".join(str(ord(c)) for c in chr(cp).lower()) + lower.update(("%d:%s;" % (cp, mapped)).encode("ascii")) + out["python_lower"] = lower.hexdigest() + + from laya import lang + + script = hashlib.sha256() + for cp in range(0x110000): + script.update(((lang._script_of(chr(cp)) or "") + ";").encode("ascii")) + out["script_of"] = script.hexdigest() + return out + + +def _round_to_int_cases(): + """Python's one-argument `round` on halfway values, and on the products `_analyse_text` forms. + + Pinned at the function level and not through the corpus, because the rounding MODE is + currently UNOBSERVABLE through `analyse`: the rounded value only ever feeds + `n_non_latin >= NON_LATIN_MIN_LETTERS`, half-to-even and half-up differ only at an exact + k + 0.5 with k even, and at an EVEN threshold both land on the same side of it. It becomes + observable the moment that threshold is odd -- at 9, a product of 8.5 is 8 half-to-even and 9 + half-up. A mutant that swapped the mode survived all of the corpus, so the contract is + recorded here instead of being left to a coincidence in a table. + """ + values = [k + 0.5 for k in range(0, 24)] + values += [-(k + 0.5) for k in range(0, 5)] + values += [0.0, 1.0, 9.0, 10.0, 0.49999999999999994, 9.499999999999998, 10.000000000000002] + for fraction in (0.1, 0.1234, 0.2, 0.25, 0.3333, 0.5, 0.75, 0.9999, 0.0312): + for letters in (0, 1, 9, 10, 17, 20, 33, 40, 85, 100, 4000): + values.append(round(fraction, 4) * letters) + out, seen = [], set() + for value in values: + key = repr(value) + if key in seen: + continue + seen.add(key) + out.append([value, round(value)]) + return out + + +def lang_detect(): + """Every output `laya.lang` produces, over a corpus built to break a port. + + WHY EACH CASE IS HERE. The corpus is not a sample of real traffic; every entry exercises a + branch that a reimplementation gets wrong by default: + + * the four regexes, which no other language spells the same way -- Java's `\\w` admits + combining marks that Python's does not, and `String.split` drops trailing empty fields + where Python's keeps them; + * `str.lower()`, which is not `String.toLowerCase` (final sigma) and not one-to-one (U+0130); + * `str.isspace()`, which `Character.isWhitespace` disagrees with on four code points, two of + them the no-break spaces a pasted ticket is full of; + * `round(x, 4)` and `round(x)`, both of which are half-to-EVEN on the exact binary value; + * dict insertion order, which decides the script profile's order and every tie between two + languages with the same score; + * and the structured-state scans, where a long English stack trace must not outvote a short + foreign message, and a name field must not pull an English ticket off the English model. + + Each case records the full output of `state_text`, `detect_script`, `script_profile`, + `latin_profile`, `analyse` and `is_english`. String cases also record every internal helper, + so a failure names the step that broke rather than only the verdict. + """ + from laya import lang + + long_english = ("The customer reports that the dashboard will not load after the update. " + "We have asked for a screenshot and the browser console output. ") * 40 + cases = [ + ("empty", ""), + ("whitespace-only", " \t\n "), + ("nbsp-only", "\u00a0\u00a0"), + ("plain-english", "I cannot log in to my account and the password reset email never arrives."), + ("english-short", "reset password"), + ("english-one-word", "refund"), + ("english-three-words", "please reset this"), + ("english-with-cafe", "We met at the cafe near the office and the invoice was paid."), + ("english-with-accented-loanword", "We met at the caf\u00e9 near the office and the invoice was paid."), + ("english-with-three-accents", "The caf\u00e9 r\u00e9sum\u00e9 of Jos\u00e9 was attached to the ticket."), + ("english-rescue-two-en-only-words", "Please the invoice caf\u00e9 and we will have it."), + ("french", "Je ne peux pas me connecter a mon compte et le mot de passe ne fonctionne pas."), + ("french-accented", "Je n'arrive pas \u00e0 me connecter \u00e0 mon compte, le mot de passe est refus\u00e9."), + ("spanish", "No puedo iniciar sesion en mi cuenta y la contrasena no funciona para nada."), + ("spanish-accented", "No puedo iniciar sesi\u00f3n en mi cuenta y la contrase\u00f1a no funciona."), + ("portuguese-stripped", "Voce pode me mandar a nota fiscal do pedido que eu fiz ontem?"), + ("portuguese-accented", "Voc\u00ea pode me mandar a nota fiscal do pedido que eu fiz ontem?"), + ("portuguese-jargon-english", "Deu erro 500 no endpoint de login depois do update de ontem"), + ("italian", "Non riesco ad accedere al mio account e la password non funziona piu"), + ("italian-articulated", "Ho ricevuto la fattura nel mese di marzo ma non trovo il pagamento"), + ("german", "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert."), + ("german-der-twice", "reduzieren der helligkeit der lichter"), + ("dutch", "Het is niet mogelijk om in te loggen op mijn account met dit wachtwoord"), + ("swedish", "Jag kan inte logga in pa mitt konto och jag behover hjalp med losenord"), + ("swedish-login-phrase", "kan inte logga in"), + ("swedish-short-fragment", "glomt losenord"), + ("swedish-one-short-word", "losenord"), + ("danish-nordic-overlap", "Hej jeg kan ikke komme ind pa min konto"), + ("romanian", "Nu pot sa intru in contul meu si parola nu functioneaza pentru care am"), + ("romanian-diacritics", "Nu pot s\u0103 intru \u00een contul meu \u0219i parola nu func\u021bioneaz\u0103"), + ("banglish", "ami amar account e login korte parchi na, password ta kaj korche na"), + ("azerbaijani", "M\u0259n hesabima daxil ola bilmir\u0259m v\u0259 \u015fifr\u0259 i\u015fl\u0259mir, bu \u00fc\u00e7\u00fcn k\u00f6m\u0259k"), + ("polish-no-stoplist", "Nie mog\u0119 si\u0119 zalogowa\u0107 na swoje konto, has\u0142o nie dzia\u0142a"), + ("turkish-no-stoplist", "Hesab\u0131ma giri\u015f yapam\u0131yorum ve \u015fifre \u00e7al\u0131\u015fm\u0131yor, yard\u0131m"), + ("turkish-dotted-capital-I", "\u0130stanbul \u0130zmir \u0130ngilizce"), + ("shared-words-only", "la o un de pe ca e que"), + ("hindi", "\u092e\u0948\u0902 \u0905\u092a\u0928\u0947 \u0916\u093e\u0924\u0947 \u092e\u0947\u0902 \u0932\u0949\u0917 \u0907\u0928 \u0928\u0939\u0940\u0902 \u0915\u0930 \u092a\u093e \u0930\u0939\u093e \u0939\u0942\u0902"), + ("korean", "\uacc4\uc815\uc5d0 \ub85c\uadf8\uc778\ud560 \uc218 \uc5c6\uc5b4\uc694"), + ("japanese", "\u30a2\u30ab\u30a6\u30f3\u30c8\u306b\u30ed\u30b0\u30a4\u30f3\u3067\u304d\u307e\u305b\u3093"), + ("chinese", "\u6211\u65e0\u6cd5\u767b\u5f55\u6211\u7684\u8d26\u6237"), + ("arabic", "\u0644\u0627 \u0623\u0633\u062a\u0637\u064a\u0639 \u062a\u0633\u062c\u064a\u0644 \u0627\u0644\u062f\u062e\u0648\u0644"), + ("hebrew", "\u05d0\u05e0\u05d9 \u05dc\u05d0 \u05d9\u05db\u05d5\u05dc \u05dc\u05d4\u05ea\u05d7\u05d1\u05e8"), + ("greek", "\u0394\u03b5\u03bd \u03bc\u03c0\u03bf\u03c1\u03ce \u03bd\u03b1 \u03c3\u03c5\u03bd\u03b4\u03b5\u03b8\u03ce"), + ("cyrillic", "\u042f \u043d\u0435 \u043c\u043e\u0433\u0443 \u0432\u043e\u0439\u0442\u0438 \u0432 \u0441\u0432\u043e\u0439 \u0430\u043a\u043a\u0430\u0443\u043d\u0442"), + ("thai", "\u0e09\u0e31\u0e19\u0e40\u0e02\u0e49\u0e32\u0e2a\u0e39\u0e48\u0e23\u0e30\u0e1a\u0e1a\u0e44\u0e21\u0e48\u0e44\u0e14\u0e49"), + ("tamil", "\u0b8e\u0ba9\u0bcd \u0b95\u0ba3\u0b95\u0bcd\u0b95\u0bbf\u0bb2\u0bcd \u0b89\u0bb3\u0bcd\u0ba8\u0bc1\u0bb4\u0bc8\u0ba3 \u0bae\u0bc1\u0b9f\u0bbf\u0baf\u0bb5\u0bbf\u0bb2\u0bcd\u0bb2\u0bc8"), + ("amharic", "\u12a0\u1230\u120b\u121d \u12a8\u1218\u1308\u1263\u1275 \u12a0\u120d\u127d\u120d\u121d"), + ("georgian", "\u10d5\u10d4\u10e0 \u10d5\u10d0\u10ee\u10d4\u10e0\u10ee\u10d4 \u10d0\u10dc\u10d2\u10d0\u10e0\u10d8\u10e8\u10d8"), + ("armenian", "\u0535\u057d \u0579\u0565\u0574 \u056f\u0561\u0580\u0578\u0572\u0561\u0576\u0578\u0582\u0574 \u0574\u057f\u0576\u0565\u056c"), + ("unlisted-script-kawi", "\U00011f00\U00011f01\U00011f02\U00011f03\U00011f04"), + ("cjk-extension-b", "\U00020000\U00020001\U00020002"), + ("fullwidth-latin", "\uff28\uff45\uff4c\uff4c\uff4f \uff37\uff4f\uff52\uff4c\uff44"), + ("latin-ext-additional", "\u1e9e\u1ebd\u1ec5\u1e0d\u1e25"), + ("ipa-pronunciation", "The name is pronounced [vl\u0250\u02c8d\u02b2im\u02b2\u0268r] in Russian and that is all"), + ("greek-symbol-in-english", "Set \u03b1 to 0.05 and then re-run the whole evaluation again"), + ("cyrillic-proper-name", "The reviewer was \u0414\u043c\u0438\u0442\u0440\u0438\u0439 \u041f\u0435\u0442\u0440\u043e\u0432\u0438\u0447 and he approved the change"), + ("cyrillic-name-with-combining", "The reviewer was \u0412\u043b\u0430\u0434\u0438\u0301\u043c\u0438\u0440 and he approved the change"), + ("latin-brand-in-cjk", "ACME-ORDER-99281-XYZ \u6ce8\u6587\u304c\u5c4a\u304d\u307e\u305b\u3093"), + ("latin-plurality-over-cjk", "Order ACME-99281 shipped but the label is wrong \u6ce8\u6587\u756a\u53f7\u304c\u9055\u3044\u307e\u3059\u3088"), + ("urls-and-emails", "See github.com/acme/repo and mail user@acme.com about v1.2.3 in the U.S.A."), + ("url-heavy-portuguese-looking", "github.com e os.path com o.com de.com na.com"), + ("leading-dot-identifier", ".com .org .net and the rest of the list is here now"), + ("at-after-dot", "a.@b and the rest of this sentence is plain English prose here"), + ("sentence-final-period", "Il pacco e arrivato. Non trovo la fattura nel portale adesso."), + ("code-line", "result = round(el, 2); os.path.join(a, b)"), + ("acronym-heavy-english", "The MON LA EST COM DES game was moved to the next week entirely"), + ("all-caps-portuguese", "N\u00c3O CONSIGO ENTRAR NA MINHA CONTA E A SENHA N\u00c3O FUNCIONA"), + ("joined-compound-names", "Nav/Com and OS/2 and C:\\DOS\\mode were all on the same list"), + ("nbsp-joined-words", "Je\u00a0ne\u00a0peux\u00a0pas\u00a0me\u00a0connecter\u00a0a\u00a0mon\u00a0compte\u00a0du\u00a0tout"), + ("final-sigma", "\u0394\u0395\u039d \u039c\u03a0\u039f\u03a1\u03a9 \u039d\u0391 \u03a3\u03a5\u039d\u0394\u0395\u0398\u03a9"), + ("english-with-portuguese-line", + "Customer ticket #4471\nNao consigo entrar na minha conta e a senha nao funciona\nAgent: asked for a screenshot"), + ("english-stack-trace-with-german-line", + "Traceback (most recent call last):\n File \"app.py\", line 12, in handler\n raise ValueError(x)\nIch kann mich nicht in mein Konto einloggen und das Passwort geht nicht"), + ("single-line-english-only", "I cannot log in to my account at all today"), + ("short-lines-under-seven", "ok\nno\nyes\nfail\nabcdef"), + # Each of the six below exists because a mutant survived without it. They are not extra + # samples of traffic; each one is the single discriminating input for one decision. + # + # 32 letters of which one is Greek: a non-Latin share of exactly 1/32 = 0.03125, which at + # four decimals is a halfway case. Python's round gives 0.0312 and the + # `Math.round(x * 1e4) / 1e4` spelling laya-ts uses gives 0.0313. + ("one-greek-letter-in-thirty-two", "Set the threshold values to \u0391 and rerun"), + # The same halfway case on the other rounded field: one diacritic in exactly 32 characters. + ("one-diacritic-in-thirty-two", "caf\u00e9 " + "x" * 27), + # Four scripts in the profile, so its ORDER is observable. With fewer than two non-Latin + # keys an unordered map passes, which is how a HashMap here survived. + ("three-non-latin-scripts", "\u0391\u0392 \u0411\u0412 \u6f22\u5b57 and some latin words here"), + # Three Latin letters against three Greek ones. The counts tie, and the tie-break is that + # Latin is inserted LAST, so the named script wins -- which only holds while the mapping + # keeps insertion order. + ("latin-and-greek-tied", "abc \u0391\u0392\u0393"), + # `todo` is an English word as well as a Spanish one, so it counts once however often it + # repeats. Counting both occurrences scores Spanish 2, clears the margin, and sends this + # to the multilingual checkpoint; counting it once scores 1 and leaves it undecided. + ("collision-word-twice", "todo todo bueno bueno"), + # German scores 2 against English's 1, which is a margin of exactly one. The reference + # requires two, so this is English; a margin of one would call it German. + ("margin-exactly-one-over-english", "der nicht the house"), + # The branch carrying the sharpest comment in the reference: English wins the whole-text + # read, so the line-by-line scan runs, and one line alone names a foreign language. These + # are the only cases that can produce a `mixed_segment`, and they are plain strings on + # purpose -- a string skips the per-leaf scan, so the segment scan is the only thing that + # can set it and a port that skipped the scan entirely would still fail here. + ("english-note-with-portuguese-line", + "The customer opened this ticket yesterday and we have asked for a screenshot.\n" + "The browser console shows no errors and the network tab looks clean to me.\n" + "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\n" + "We will escalate this to the platform team if it is not resolved today."), + ("portuguese-line-first-then-english", + "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\n" + "The customer opened this ticket yesterday and we have asked for a screenshot.\n" + "The browser console shows no errors and the network tab looks clean to me.\n" + "We will escalate this to the platform team if it is not resolved today."), + ("english-note-with-german-line", + "The customer opened this ticket yesterday and we have asked for a screenshot.\n" + "The browser console shows no errors and the network tab looks clean to me.\n" + "Ich kann mich nicht in mein Konto einloggen und das Passwort wird nicht akzeptiert\n" + "We will escalate this to the platform team if it is not resolved today."), + ("english-note-with-code-line-and-foreign-line", + "The customer opened this ticket yesterday and we have asked for a screenshot.\n" + "result = round(el, 2); os.path.join(a, b)\n" + "Non riesco ad accedere al mio account nel portale e la fattura non arriva\n" + "We will escalate this to the platform team if it is not resolved today."), + ("english-note-with-acronym-line-and-foreign-line", + "The customer opened this ticket yesterday and we have asked for a screenshot.\n" + "MON LA EST COM DES\n" + "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum\n" + "We will escalate this to the platform team if it is not resolved today."), + ] + structured = [ + ("none-state", None), + ("number-state", 42), + ("bool-state", True), + ("empty-dict", {}), + ("empty-list", []), + ("bytes-valid", "N\u00e3o consigo entrar na minha conta e a senha n\u00e3o funciona".encode("utf-8")), + ("bytes-invalid", b"\xff\xfe not utf-8 at all"), + ("dict-english-note-and-portuguese-message", { + "note": long_english, + "message": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum", + }), + ("dict-portuguese-past-the-cap", { + "note": long_english, + "tail": "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum", + }), + ("dict-english-name-field", {"name": "Jos\u00e9", "body": "Please reset my password for this account today"}), + ("dict-cyrillic-name-field", {"name": "\u0414\u043c\u0438\u0442\u0440\u0438\u0439", "body": "Please reset my password for this account today"}), + ("dict-all-english", {"a": "Please reset my password", "b": "The dashboard will not load at all"}), + ("nested-list-of-dicts", [{"t": "Please reset my password for the account"}, + {"t": "Ich kann mich nicht in mein Konto einloggen und das Passwort"}]), + ("deep-nesting-within-limit", [[[[[["Ich kann mich nicht in mein Konto einloggen und das Passwort"]]]]]]), + ("deep-nesting-past-limit", [[[[[[["Ich kann mich nicht in mein Konto einloggen und das Passwort"]]]]]]]), + ("dict-with-korean-value", {"id": "ORDER-1", "msg": "\uacc4\uc815\uc5d0 \ub85c\uadf8\uc778\ud560 \uc218 \uc5c6\uc5b4\uc694 \ub3c4\uc640\uc8fc\uc138\uc694"}), + ("dict-with-code-leaf", {"trace": "result = round(el, 2);\nos.path.join(a, b)", + "msg": "Please take a look at this when you can"}), + ("list-of-numbers", [1, 2.5, None, True]), + ("long-single-leaf-over-budget", long_english), + ("two-leaves-budget-split", ["x" * 3990, "Nao consigo entrar na minha conta e a senha nao funciona"]), + # A short English note leaves the segment scan enough budget to reach the second field, so + # here the segment scan names the language and records the field. Its sibling above, whose + # note is long enough to exhaust the budget, is answered by the per-leaf scan instead and + # records no segment -- the two together pin both halves of #384. + ("dict-short-english-note-and-german-message", { + "note": ("The customer opened this ticket yesterday and we have asked for a " + "screenshot. The browser console shows no errors and the network tab " + "looks clean to me. We will escalate this to the platform team today. "), + "message": "Ich kann mich nicht in mein Konto einloggen und das Passwort geht nicht", + }), + ] + + def record(name, state): + text = lang.state_text(state) + profile = lang.latin_profile(text) + analysis = lang.analyse(state) + row = { + "name": name, + "state": _state_repr(state), + "state_text": text, + "detect_script": lang.detect_script(text), + "script_profile": lang.script_profile(text), + "latin_profile": { + "language": profile["language"], + "english_hits": profile["english_hits"], + "diacritic_rate": profile["diacritic_rate"], + "looks_non_english": profile["looks_non_english"], + }, + "analyse": { + "script": analysis["script"], + "script_profile": analysis["script_profile"], + "language": analysis["language"], + "is_english": analysis["is_english"], + "language_undecided": analysis["language_undecided"], + "diacritic_rate": analysis["diacritic_rate"], + "non_latin_fraction": analysis["non_latin_fraction"], + "mixed_segment": analysis["mixed_segment"], + }, + "is_english": lang.is_english(state), + } + if isinstance(state, str): + blanked = lang._LETTER_RUN.sub( + lambda m: " " if m.group().isupper() else m.group(), state) + row["helpers"] = { + "words": lang._WORD.findall(state), + "substitute_identifiers": lang._IDENTIFIER.sub(" ", state), + "non_latin_words": lang._non_latin_words(state), + "named_prose_language": lang._named_prose_language(state), + "has_code_line": bool(lang._CODE_LINE.search(state)), + "has_joined_tokens": [bool(lang._JOINED.search(tok)) for tok in state.split()], + "blank_upper_runs": blanked, + "python_lower": state.lower(), + "strip": state.strip(), + "split_whitespace": state.split(), + "split_newline": state.split("\n"), + "count_alpha": sum(ch.isalpha() for ch in state), + "code_point_length": len(state), + "english_rescued_by_words": lang._english_rescued_by_words( + lang._WORD.findall(lang._IDENTIFIER.sub(" ", state).replace("\u0130", "i").lower()), + profile["diacritic_rate"]), + } + return row + + return { + "notes": lang_detect.__doc__, + "thresholds": { + "non_en_diacritic_rate": lang.NON_EN_DIACRITIC_RATE, + "english_rescue_diacritic_rate": lang.ENGLISH_RESCUE_DIACRITIC_RATE, + "non_latin_fraction": lang.NON_LATIN_FRACTION, + "non_latin_min_fraction": lang.NON_LATIN_MIN_FRACTION, + "non_latin_min_letters": lang.NON_LATIN_MIN_LETTERS, + }, + "unicode_version": unicodedata.unidata_version, + "unicode_digests": _unicode_digests(), + "round_to_int": _round_to_int_cases(), + "cases": [record(name, state) for name, state in cases] + + [record(name, state) for name, state in structured], + } + def presets(): """The preset question dicts exactly as a caller receives them, and as the model is shown them. @@ -693,6 +1049,7 @@ def predict_golden(): FAMILIES = { "lang_tables.json": lang_tables, + "lang_detect.json": lang_detect, "presets.json": presets, "tokenizer_ids.json": tokenizer_ids, "sequences.json": sequences, diff --git a/laya-java/scripts/gen_language_tables.py b/laya-java/scripts/gen_language_tables.py index 8062a0b17..e886d5d33 100755 --- a/laya-java/scripts/gen_language_tables.py +++ b/laya-java/scripts/gen_language_tables.py @@ -45,6 +45,20 @@ def quote(text): return "".join(out) +def wrap_items(items, indent=" ", width=104): + """Comma-separated items wrapped to a readable width, as Java array/argument contents.""" + lines, current = [], indent + for item in items: + piece = item + ", " + if len(current) + len(piece) > width and current.strip(): + lines.append(current.rstrip()) + current = indent + current += piece + if current.strip(): + lines.append(current.rstrip().rstrip(",")) + return "\n" + "\n".join(lines) + "\n " + + def words_block(name, words, doc, sort=True): items = sorted(words) if sort else list(words) lines, current = [], " " @@ -69,6 +83,26 @@ def render(): pairs = ", ".join("%d, %d" % (lo, hi) for lo, hi in ranges) script_rows.append(' new Script(%s, new int[] {%s})' % (quote(name), pairs)) + # The flat lookup emitted below replaces the ordered scan with a binary search. The two are + # equivalent only while no two ranges overlap: where they do, the first-match rule decides and + # a table sorted by code point cannot see it. Checked here rather than trusted, so an edit to + # `_SCRIPT_RANGES` that introduces an overlap fails this generator instead of quietly changing + # which script a code point is counted under. + flat = sorted((lo, hi, name) for name, ranges in lang._SCRIPT_RANGES for lo, hi in ranges) + for (a_lo, a_hi, a_name), (b_lo, b_hi, b_name) in zip(flat, flat[1:]): + if a_hi >= b_lo: + raise SystemExit( + "gen_language_tables: script ranges %s [%04X-%04X] and %s [%04X-%04X] overlap, so " + "the flat lookup would lose the first-match rule. Drop SCRIPT_LOOKUP back to an " + "ordered scan, or remove the overlap." + % (a_name, a_lo, a_hi, b_name, b_lo, b_hi)) + lookup_pairs = wrap_items(["%d, %d" % (lo, hi) for lo, hi, _ in flat]) + distinct = [name for name, _ in lang._SCRIPT_RANGES] + index_of = {name: n for n, name in enumerate(distinct)} + lookup_index = wrap_items([str(index_of[name]) for _, _, name in flat]) + script_names = wrap_items([quote(name) for name in distinct]) + script_max = max(hi for _, hi, _ in flat) + stop_rows = [] for code, words in lang._STOP.items(): items = ", ".join(quote(w) for w in sorted(words)) @@ -104,8 +138,22 @@ def render(): private LanguageTables() { } - /** One named non-Latin script and the code-point ranges that belong to it. */ + /** + * One named non-Latin script and the code-point ranges that belong to it. + * + *

The array is copied in and out, so a table a caller holds cannot be edited under + * detection. Nothing on the hot path reads it -- {@link #lookupScript} does -- so the copy + * costs nothing that matters. + */ public record Script(String name, int[] ranges) { + public Script { + ranges = ranges.clone(); + } + + @Override + public int[] ranges() { + return ranges.clone(); + } } ''' @@ -116,7 +164,77 @@ def render(): ' */\n' ' public static final List