Strip BOM from decoded unicode content

Author: User

tests/test_bom_encoding.py

@@ -1,37 +1,48 @@
 # coding: utf-8
 import codecs
 
-from unicodec.bom_encoding import find_bom_encoding
+from unicodec.bom_encoding import BOM_ENCODINGS, find_bom_encoding
+from unicodec.main import decode_content
 
 
-def test_bom_preservation_while_decoding():
-    # type: () -> None
+def test_decode_utf8_preserves_bom():  # type: () -> None
     # fmt: off
     msg = codecs.BOM_UTF8 + u"привет".encode("utf-8")
     assert msg.decode("utf-8") == codecs.BOM_UTF8.decode("utf-8") + u"привет"
     # fmt: on
 
 
-def test_find_bom_encoding_empty_data():
-    # type: () -> None
-    assert find_bom_encoding(b"") is None
+def test_find_bom_encoding_empty_data():  # type: () -> None
+    assert find_bom_encoding(b"") == (None, None)
 
 
-def test_find_bom_encoding_nonbom_data():
-    # type: () -> None
-    assert find_bom_encoding(b"asdf") is None
+def test_find_bom_encoding_nonbom_data():  # type: () -> None
+    assert find_bom_encoding(b"asdf") == (None, None)
 
 
-def test_find_bom_encoding_utf8_bom():
-    # type: () -> None
-    assert find_bom_encoding(codecs.BOM_UTF8 + b"asdf") == "utf-8"
+def test_find_bom_encoding_utf8_bom():  # type: () -> None
+    assert find_bom_encoding(codecs.BOM_UTF8 + b"asdf") == (codecs.BOM_UTF8, "utf-8")
 
 
-def test_find_bom_encoding_utf16_be_bom():
-    # type: () -> None
-    assert find_bom_encoding(codecs.BOM_UTF16_BE + b"asdf") == "utf-16-be"
+def test_find_bom_encoding_utf16_be_bom():  # type: () -> None
+    assert find_bom_encoding(codecs.BOM_UTF16_BE + b"asdf") == (
+        codecs.BOM_UTF16_BE,
+        "utf-16-be",
+    )
 
 
-def test_find_bom_encoding_utf32_le_bom():
-    # type: () -> None
-    assert find_bom_encoding(codecs.BOM_UTF32_LE + b"asdf") == "utf-32-le"
+def test_find_bom_encoding_utf32_le_bom():  # type: () -> None
+    assert find_bom_encoding(codecs.BOM_UTF32_LE + b"asdf") == (
+        codecs.BOM_UTF32_LE,
+        "utf-32-le",
+    )
+
+
+def test_bom_is_stripped():  # type: () -> None
+    # fmt: off
+    msg = u"жук"
+    # fmt: on
+    for bom, enc in BOM_ENCODINGS:
+        msg_enc = bom + msg.encode(enc)
+        msg_dec = decode_content(msg_enc)
+        assert msg_dec == msg

unicodec/bom_encoding.py

@@ -3,6 +3,9 @@
 __all__ = ["find_bom_encoding"]
 
 # Order does matter here. UTF-32 BOMs must be preceed UTF-16 BOMs.
+# fmt: off
+BOM_UNICODE = u'\ufeff'
+# fmt: on
 BOM_ENCODINGS = [
     (codecs.BOM_UTF32_BE, "utf-32-be"),
     (codecs.BOM_UTF32_LE, "utf-32-le"),
@@ -13,13 +16,13 @@
 
 
 def find_bom_encoding(data):
-    # type: (bytes) -> None | str
+    # type: (bytes) -> tuple[None, None] | tuple[bytes, str]
     """Search for BOM signature and return encoding which uses such bom.
 
     Return None if BOM signature not found.
     """
     # Reference: https://encoding.spec.whatwg.org/#bom-sniff
     for bom_bytes, enc in BOM_ENCODINGS:
         if data.startswith(bom_bytes):
-            return enc
-    return None
+            return bom_bytes, enc
+    return None, None

unicodec/main.py

@@ -4,7 +4,7 @@
 from typing import Literal
 
 from . import entities
-from .bom_encoding import find_bom_encoding
+from .bom_encoding import BOM_UNICODE, find_bom_encoding
 from .errors import InvalidEncodingNameError
 from .html_encoding import detect_html_encoding
 from .http_encoding import parse_content_type_header_encoding
@@ -20,7 +20,7 @@ def detect_content_encoding(
     markup="html",  # type: Literal["html", "xml"]
 ):
     # type: (...) -> str
-    enc = find_bom_encoding(data)
+    bom, enc = find_bom_encoding(data)
     if enc:
         try:
             return normalize_encoding_name(enc)
@@ -57,6 +57,9 @@ def decode_content(  # pylint: disable=R0917
                 data, content_type_header=content_type_header, markup=markup
             )
         data = data.decode(encoding)
+    # Remove BOM, it might be at the start of decoded unicode text
+    if data.startswith(BOM_UNICODE):
+        data = data[len(BOM_UNICODE) :]
     if remove_null_bytes:
         data = data.replace("\x00", "")
     if decode_entities: