diff --git a/laya-ts/src/email.ts b/laya-ts/src/email.ts index 09c1cc245..2959b7a1f 100644 --- a/laya-ts/src/email.ts +++ b/laya-ts/src/email.ts @@ -19,9 +19,9 @@ const HEADER_NEXT = /^\s*(Enviad[oa]( em| el)?:\s|(Data|Fecha):\s.*\d{4})/i; // case-insensitively here and the name is checked case-sensitively by SIGNOFF_TAIL. const SIGNOFF_HEAD = /^\s*(?:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)(?:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?/i; -// Python's name class [^\W\d_a-zß-öø-ÿ] is "a word char that is not a digit, underscore or -// lowercase letter"; \p{Lu}/\p{Lt}/\p{Lo} is the same intent: a name is capitalised in any -// script (Regards, Łukasz) or written in a script without case (山田). +// The name's first letter must not be lowercase, so the check is per character, not per script: +// \p{Lu}/\p{Lt}/\p{Lo} accepts a name capitalised in any script (Regards, Łukasz) or written in +// a script without case (山田). Python asks the same question per token (`_is_english_signoff`). const SIGNOFF_TAIL = /^[\s,;:!.]*(?:[\p{Lu}\p{Lt}\p{Lo}][\p{L}\p{M}\p{N}_'-]*[\s,.]*){0,3}$/u; function isEnglishSignoff(line: string): boolean { const m = SIGNOFF_HEAD.exec(line); diff --git a/laya/email.py b/laya/email.py index a4a6dcf20..fd7f222ce 100644 --- a/laya/email.py +++ b/laya/email.py @@ -7,6 +7,7 @@ history (often a *different* request) weighed on the answer as much as the new message did. """ import re +import unicodedata from typing import Dict, List, Optional # One definition, in the module that holds the other presets. Re-exported here because @@ -45,20 +46,49 @@ # English pair is "From: " followed by "Sent:"/"Date:". _HEADER_FROM_NAME = re.compile(r"^\s*(De|From):\s+\S", re.I) _HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})", re.I) +# A closing's name starts with a letter that is not lowercase: capitalised in any script +# (`Łukasz`, `Дмитрий`) or caseless (`山田`). `re` cannot say "not a lowercase letter in any +# script" -- a class has to list ranges, and `[^\W\d_a-zß-öø-ÿ]` stops at Latin-1, so +# `Thanks, żaneta` read as a name and the line counted as a sign-off. The tail is matched +# structurally instead, and each token's first letter is judged by category below -- the same +# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before +# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port. +_SIGNOFF_HEAD = re.compile( + r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)" + r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?" +) +_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w\u0300-\u036f'-]*[\s,.]*){0,3}$") +_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w\u0300-\u036f'-]*") + + +def _is_english_signoff(line: str) -> bool: + """True when a closing word is followed by nothing but punctuation and a short name.""" + m = _SIGNOFF_HEAD.match(line) + if m is None: + return False + tail = line[m.end():] + if _SIGNOFF_TAIL.match(tail) is None: + return False + return all(unicodedata.category(token[0]) in ("Lu", "Lt", "Lo") + for token in _SIGNOFF_TOKEN.findall(tail)) + + +def _marker_matches(marker, line: str) -> bool: + """One `_SIGNATURE_MARKERS` entry: a compiled pattern, or a callable for a rule a pattern + cannot express (the English sign-off).""" + return bool(marker(line)) if callable(marker) else bool(marker.match(line)) + + _SIGNATURE_MARKERS = [ re.compile(r"^\s*--\s*$"), # A closing line is the closing word plus punctuation and at most a name. Anything else on # the line is a sentence, and the case of the next word is what separates the two: a name is # capitalised, "for" in "Thanks for the quick reply." is not. The closing words are matched # case-insensitively, the name is not, so the flag is scoped instead of global. - # `warmest` and `and/& regards` are closings the alternation did not reach, and a name is - # capitalised in any script, so the name class excludes the lowercase letters instead of - # listing the uppercase ones: `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not. - re.compile( - r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)" - r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?" - r"[\s,;:!.]*(?:[^\W\d_a-zß-öø-ÿ][\w'-]*[\s,.]*){0,3}$" - ), + # `warmest` and `and/& regards` are closings the alternation did not reach; the name that may + # follow is judged in `_is_english_signoff` above, a callable because `re` cannot express its + # rule. `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not. + _is_english_signoff, re.compile(r"^\s*sent from my (iphone|android|mobile|ipad)", re.I), # Portuguese/Spanish sign-offs match only on their own: "Obrigado pelo retorno, mas ..." is a # request, not a signature, so unlike the English marker no trailing words are allowed @@ -185,7 +215,7 @@ def clean_email_body(body: str, max_chars: int = 3000) -> str: cut = len(lines) for i in range(max(1, min(int(len(lines) * 0.6), len(lines) - 8)), len(lines)): n = len(lines[i].strip()) - if (n <= 40 and any(p.match(lines[i]) for p in _SIGNATURE_MARKERS)) or ( + if (n <= 40 and any(_marker_matches(p, lines[i]) for p in _SIGNATURE_MARKERS)) or ( n <= 60 and _DEVICE_FOOTER.match(lines[i])): cut = i break diff --git a/tests/test_email.py b/tests/test_email.py index 0d13424a2..000817080 100644 --- a/tests/test_email.py +++ b/tests/test_email.py @@ -426,6 +426,28 @@ def check_true(name, cond, detail=""): ]: check("signoff kept/" + label, clean_email_body(body), body) +# ------------------------------------------- the name rule reaches lowercase beyond Latin-1 +# #245 replaced the ASCII `[A-Z]` with "exclude the lowercase letters", but the ranges that +# exclude them stop at Latin-1 (`a-zß-öø-ÿ`). A word starting with ż, д or α *is* lowercase -- +# not "capitalised in any script" -- yet the class read it as a name, so the line counted as a +# sign-off and the cut dropped whatever followed it. Case is a property of the character, not of +# the script: the port checks it per character (`\p{Lu}\p{Lt}\p{Lo}`), and Python now checks the +# same first-letter categories. +for label, body in [ + ("latin-extended lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, żaneta"), + ("cyrillic lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, дмитрий"), + ("greek lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, αλέξανδρος"), + ("a roman numeral is not a name", "Hi,\n\nPlease refund invoice 4411.\nThanks, Ⅷ"), +]: + check("signoff kept/" + label, clean_email_body(body), body) +# ...while a combining mark stays part of the letter before it: `Jose\u0301` spells `José` and +# still cuts as a sign-off (the port's `\p{M}`). +check( + "signoff cut/decomposed accented name", + clean_email_body("%s\n\n%s" % (BODY, "Regards, Jose\u0301")), + BODY, +) + # ------------------------------------------------- the word, without the disclaimer # `confidential` was a bare substring of `_DISCLAIMER`, so any sentence that merely