exclude vendored spdx data from sdist/whl. build/bring our own

Per feedback, integrate a variant of #799 that builds a minimal JSON dataset to feed vendored license-expression

32K	src/packaging/_spdx.json

vs

848K	src/packaging/_vendor/license_expression/data/scancode-licensedb-index.json

Author: ewdurbin

pyproject.toml

@@ -37,7 +37,7 @@ Source = "https://github.com/pypa/packaging"
 
 [tool.flit.sdist]
 include = ["LICENSE*", "tests/", "docs/", "CHANGELOG.rst"]
-exclude = ["docs/_build", "tests/manylinux/build-hello-world.sh", "tests/musllinux/build.sh", "tests/hello-world.c", "tests/__pycache__", "build/__pycache__"]
+exclude = ["docs/_build", "tests/manylinux/build-hello-world.sh", "tests/musllinux/build.sh", "tests/hello-world.c", "tests/__pycache__", "build/__pycache__", "src/packaging/_vendor/license_expression/data/*"]
 
 [tool.pytest.ini_options]
 addopts = [

src/packaging/_spdx.json

@@ -6,6 +6,7 @@
 import email.message
 import email.parser
 import email.policy
+import importlib.resources
 import typing
 from typing import (
     Any,
@@ -647,7 +648,8 @@ def _process_requires_dist(
             return reqs
 
     def _process_license_expression(self, value: str) -> str:
-        licensing = get_spdx_licensing()
+        with importlib.resources.path("packaging", "_spdx.json") as spdx_path:
+            licensing = get_spdx_licensing(license_index_location=spdx_path)
         try:
             return str(licensing.parse(value, validate=True))
         except LicenseExpressionError as exc:

src/packaging/metadata.py

@@ -4,6 +4,7 @@
 
 import invoke
 
-from . import check
+from . import check, licenses
 
 ns = invoke.Collection(check)
+ns.add_collection(licenses)

tasks/__init__.py

@@ -0,0 +1,67 @@
+import json
+import time
+
+import httpx
+import invoke
+
+from .paths import SPDX_LICENSES
+
+LATEST_API = "https://api.github.com/repos/spdx/license-list-data/releases/latest"
+LICENSES_URL = (
+    "https://raw.githubusercontent.com/spdx/license-list-data/v{}/json/licenses.json"
+)
+EXCEPTIONS_URL = (
+    "https://raw.githubusercontent.com/spdx/license-list-data/v{}/json/exceptions.json"
+)
+
+
+def download_data(url):
+    for _ in range(600):
+        try:
+            response = httpx.get(url)
+            response.raise_for_status()
+        except Exception:
+            time.sleep(1)
+            continue
+        else:
+            return json.loads(response.content.decode("utf-8"))
+
+    message = "Download failed"
+    raise ConnectionError(message)
+
+
+@invoke.task
+def update(ctx):
+    print("Updating SPDX licenses...")
+
+    latest_version = download_data(LATEST_API)["tag_name"][1:]
+    print(f"Latest version: {latest_version}")
+
+    license_payload = download_data(LICENSES_URL.format(latest_version))["licenses"]
+    print(f"Licenses: {len(license_payload)}")
+
+    exception_payload = download_data(EXCEPTIONS_URL.format(latest_version))[
+        "exceptions"
+    ]
+    print(f"Exceptions: {len(exception_payload)}")
+
+    licenses = []
+    for license_data in license_payload:
+        _l = {
+            "spdx_license_key": license_data["licenseId"],
+        }
+        if license_data["isDeprecatedLicenseId"]:
+            _l["is_deprecated"] = license_data["isDeprecatedLicenseId"]
+        licenses.append(_l)
+
+    for exception_data in exception_payload:
+        _l = {
+            "spdx_license_key": exception_data["licenseExceptionId"],
+            "is_exception": True,
+        }
+        if exception_data["isDeprecatedLicenseId"]:
+            _l["is_deprecated"] = exception_data["isDeprecatedLicenseId"]
+        licenses.append(_l)
+
+    with open(SPDX_LICENSES, "w", encoding="utf-8") as f:
+        f.write(json.dumps(licenses))

tasks/licenses.py

@@ -7,3 +7,4 @@
 PROJECT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 
 CACHE = os.path.join(PROJECT, ".cache")
+SPDX_LICENSES = os.path.join(PROJECT, "src", "packaging", "_spdx.json")

tasks/paths.py

@@ -1,3 +1,4 @@
 # The requirements required to invoke the tasks
 invoke
 progress
+httpx