Fix: handle non-UTF-8 input in util.py

Chemistry papers such as DOIs 10.26434/chemrxiv-2025-1x058-v2 and 10.26434/chemrxiv-2025-3lwn2 contain files that sometimes throw errors due to non-UTF-8 characters during parsing. This change updates the MD5 hashing line to replace problematic characters during UTF-8 encoding, making the parsing process more robust and preventing interruptions caused by encoding errors.
Future-House · dmcgrath19 · Aug 6, 2025 · Aug 6, 2025 · Aug 6, 2025 · Aug 7, 2025
commit 687ce4023a4b7ba2a69a3f8e7611645fca6e1305
diff --git a/src/paperqa/utils.py b/src/paperqa/utils.py
@@ -104,7 +104,7 @@ def strings_similarity(s1: str, s2: str, case_insensitive: bool = True) -> float
 
 def hexdigest(data: str | bytes) -> str:
     if isinstance(data, str):
-        return hashlib.md5(data.encode("utf-8")).hexdigest()  # noqa: S324
+        return hashlib.md5(data.encode("utf-8", errors="replace")).hexdigest()  # noqa: S324
-        return hashlib.md5(data.encode("utf-8", errors="replace")).hexdigest()  # noqa: S324
+        return hashlib.md5(data.encode("utf-8", errors="strict")).hexdigest()  # noqa: S324
-        return hashlib.md5(data.encode("utf-8", errors="replace")).hexdigest()  # noqa: S324
+        return hashlib.md5(data.encode("utf-8", errors="strict")).hexdigest()  # noqa: S324
     return hashlib.md5(data).hexdigest()  # noqa: S324