From a662f822906ca49152b1b8af7855aaceebdfd33d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Feb 2026 20:56:48 +0530 Subject: [PATCH 01/26] Fix views bug Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 3603b0f60..860bde8eb 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -506,15 +506,15 @@ def get_context_data(self, **kwargs): epss_data = None epss_advisory = None if not epss_severity: - related_epss_advisory = ( + epss_advisory = ( advisory.related_advisory_severities.filter( datasource_id=EPSSImporterPipeline.pipeline_id ) .latest_per_avid() .first() ) - epss_advisory = related_epss_advisory - epss_severity = related_epss_advisory.severities.filter(scoring_system="epss").first() + if epss_advisory: + epss_severity = epss_advisory.severities.filter(scoring_system="epss").first() if epss_severity: # If the advisory itself does not have EPSS severity, but has a related advisory with EPSS severity, we use the related advisory's EPSS severity and URL as the source of EPSS data. epss_data = { From fd2040f93c6a5f3775ec94d7843f4c885e3e6584 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Feb 2026 21:02:35 +0530 Subject: [PATCH 02/26] Fix HTML views Signed-off-by: Tushar Goel --- vulnerabilities/templates/package_details_v2.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index e01f5243e..9cc9ea343 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -45,7 +45,7 @@
- {% if affected_by_advisories|length != 0 %} + {% if affected_by_advisories_v2|length != 0 %}
{% else %}
@@ -82,7 +82,7 @@
- {% if affected_by_advisories|length != 0 %} + {% if affected_by_advisories_v2|length != 0 %}
From 1334cdf65081193f38f6c25c0bae4919259768d2 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 11 Feb 2026 19:01:24 +0530 Subject: [PATCH 03/26] Add pipeline to federate package vulnerabilities Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/__init__.py | 4 + .../federate_package_vulnerabilities.py | 257 ++++++++++++++++++ vulnerabilities/pipes/federatedcode.py | 175 ++++++++++++ vulnerablecode/settings.py | 10 + 4 files changed, 446 insertions(+) create mode 100644 vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py create mode 100644 vulnerabilities/pipes/federatedcode.py diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index e563846fe..632fd95f6 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -141,6 +141,10 @@ def log(self, message, level=logging.INFO): class VulnerableCodePipeline(PipelineDefinition, BasePipelineRun): pipeline_id = None # Unique Pipeline ID + # When set to true pipeline is run only once. + # To rerun onetime pipeline reset is_active field to True via migration. + run_once = False + def on_failure(self): """ Tasks to run in the event that pipeline execution fails. diff --git a/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py new file mode 100644 index 000000000..3bc3dcbaf --- /dev/null +++ b/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py @@ -0,0 +1,257 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import itertools +import shutil +from operator import attrgetter +from pathlib import Path + +import saneyaml +from aboutcode.pipeline import LoopProgress +from django.conf import settings + +from aboutcode.federated import DataFederation +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipes import federatedcode + + +class FederatePackageVulnerabilities(VulnerableCodePipeline): + """Export package vulnerabilities and advisory to FederatedCode.""" + + pipeline_id = "federate_package_vulnerabilities_v2" + + @classmethod + def steps(cls): + return ( + cls.check_federatedcode_eligibility, + cls.create_federatedcode_working_dir, + cls.fetch_federation_config, + cls.clone_vulnerabilities_repo, + cls.publish_vulnerabilities, + cls.delete_working_dir, + ) + + def check_federatedcode_eligibility(self): + """Check if FederatedCode is configured.""" + federatedcode.check_federatedcode_configured_and_available(self.log) + + def create_federatedcode_working_dir(self): + """Create temporary working dir.""" + self.working_path = federatedcode.create_federatedcode_working_dir() + + def fetch_federation_config(self): + """Fetch config for PackageURL Federation.""" + data_federation = DataFederation.from_url( + name="aboutcode-data", + remote_root_url="https://github.com/aboutcode-data", + ) + self.data_cluster = data_federation.get_cluster("purls") + + def clone_vulnerabilities_repo(self): + self.repo = federatedcode.clone_repository( + repo_url=settings.FEDERATEDCODE_VULNERABILITIES_REPO, + clone_path=self.working_path / "vulnerabilities-data", + logger=self.log, + ) + + def publish_vulnerabilities(self): + """Publish package vulnerabilities and advisory to FederatedCode""" + repo_path = Path(self.repo.working_dir) + commit_count = 1 + batch_size = 2000 + files_to_commit = set() + exported_avids = set() + + distinct_packages_count = ( + PackageV2.objects.values("type", "namespace", "name") + .distinct("type", "namespace", "name") + .count() + ) + package_qs = package_prefetched_qs() + grouped_packages = itertools.groupby( + package_qs.iterator(chunk_size=2000), + key=attrgetter("type", "namespace", "name"), + ) + + self.log(f"Exporting vulnerabilities for {distinct_packages_count} packages.") + progress = LoopProgress( + total_iterations=distinct_packages_count, + progress_step=1, + logger=self.log, + ) + for _, packages in progress.iter(grouped_packages): + package_urls = [] + package_vulnerabilities = [] + for package in packages: + purl = package.package_url + package_urls.append(purl) + package_vulnerabilities.append(serialize_package_vulnerability(package)) + + impacts = itertools.chain( + package.affected_in_impacts.all(), + package.fixed_in_impacts.all(), + ) + for impact in impacts: + adv = impact.advisory + avid = adv.avid + if avid in exported_avids: + continue + + exported_avids.add(avid) + advisory = serialize_advisory(adv) + adv_file = f"vulnerabilities/{avid}.yml" + write_file( + repo_path=repo_path, + file_path=adv_file, + data=advisory, + ) + files_to_commit.add(adv_file) + + package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl=purl) + package_vulnerability_path = datafile_path.replace("/purls.yml", "/vulnerabilities.yml") + package_vulnerability_path = f"packages/{package_repo}/{package_vulnerability_path}" + package_path = f"packages/{package_repo}/{datafile_path}" + + write_file( + repo_path=repo_path, + file_path=package_path, + data=package_urls, + ) + files_to_commit.add(package_path) + + write_file( + repo_path=repo_path, + file_path=package_vulnerability_path, + data=package_vulnerabilities, + ) + files_to_commit.add(package_vulnerability_path) + + if len(files_to_commit) > batch_size: + if federatedcode.commit_and_push_changes( + commit_message=self.commit_message(commit_count), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ): + commit_count += 1 + files_to_commit.clear() + + if files_to_commit: + federatedcode.commit_and_push_changes( + commit_message=self.commit_message(commit_count, commit_count), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ) + + self.log( + f"Federated {distinct_packages_count} package and {len(exported_avids)} vulnerabilities." + ) + + def delete_working_dir(self): + """Remove temporary working dir.""" + if hasattr(self, "working_path") and self.working_path: + shutil.rmtree(self.working_path) + + def on_failure(self): + self.delete_working_dir() + + def commit_message(self, commit_count, total_commit_count="many"): + """Commit message for pushing Package vulnerability.""" + return federatedcode.commit_message( + commit_count=commit_count, + total_commit_count=total_commit_count, + ) + + +def package_prefetched_qs(): + return PackageV2.objects.order_by("type", "namespace", "name", "version").prefetch_related( + "affected_in_impacts", + "affected_in_impacts__advisory", + "affected_in_impacts__advisory__impacted_packages", + "affected_in_impacts__advisory__aliases", + "affected_in_impacts__advisory__references", + "affected_in_impacts__advisory__severities", + "affected_in_impacts__advisory__weaknesses", + "fixed_in_impacts", + "fixed_in_impacts__advisory", + "fixed_in_impacts__advisory__impacted_packages", + "fixed_in_impacts__advisory__aliases", + "fixed_in_impacts__advisory__references", + "fixed_in_impacts__advisory__severities", + "fixed_in_impacts__advisory__weaknesses", + ) + + +def serialize_package_vulnerability(package): + affected_by_vulnerabilities = [ + impact.advisory.avid for impact in package.affected_in_impacts.all() + ] + fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()] + + return { + "purl": package.package_url, + "affected_by_vulnerabilities": affected_by_vulnerabilities, + "fixing_vulnerabilities": fixing_vulnerabilities, + } + + +def serialize_severity(sev): + return { + "score": sev.value, + "scoring_system": sev.scoring_system, + "scoring_elements": sev.scoring_elements, + "published_at": str(sev.published_at), + "url": sev.url, + } + + +def serialize_references(reference): + return { + "url": reference.url, + "reference_type": reference.reference_type, + "reference_id": reference.reference_id, + } + + +def serialize_advisory(advisory): + """Return a plain data mapping serialized from advisory object.""" + aliases = [a.alias for a in advisory.aliases.all()] + severities = [serialize_severity(sev) for sev in advisory.severities.all()] + weaknesses = [wkns.cwe for wkns in advisory.weaknesses.all()] + references = [serialize_references(ref) for ref in advisory.references.all()] + impacts = [ + { + "purl": impact.base_purl, + "affected_versions": impact.affecting_vers, + "fixed_versions": impact.fixed_vers, + } + for impact in advisory.impacted_packages.all() + ] + + return { + "advisory_id": advisory.advisory_id, + "datasource_id": advisory.avid, + "datasource_url": advisory.url, + "aliases": aliases, + "summary": advisory.summary, + "impacted_packages": impacts, + "severities": severities, + "weaknesses": weaknesses, + "references": references, + } + + +def write_file(repo_path, file_path, data): + """Write ``data`` as YAML to ``repo_path``.""" + write_to = repo_path / file_path + write_to.parent.mkdir(parents=True, exist_ok=True) + with open(write_to, encoding="utf-8", mode="w") as f: + f.write(saneyaml.dump(data)) diff --git a/vulnerabilities/pipes/federatedcode.py b/vulnerabilities/pipes/federatedcode.py new file mode 100644 index 000000000..604c79237 --- /dev/null +++ b/vulnerabilities/pipes/federatedcode.py @@ -0,0 +1,175 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import logging +import tempfile +import textwrap +from pathlib import Path +from urllib.parse import urlparse + +import requests +from django.conf import settings +from git import GitCommandError +from git import Repo + +logger = logging.getLogger(__name__) + + +def url_exists(url, timeout=5): + """ + Check if the given `url` is reachable by doing head request. + Return True if response status is 200, else False. + """ + try: + response = requests.head(url, timeout=timeout) + response.raise_for_status() + except requests.exceptions.RequestException as request_exception: + logger.debug(f"Error while checking {url}: {request_exception}") + return False + + return response.status_code == requests.codes.ok + + +def is_configured(): + """Return True if the required FederatedCode settings have been set.""" + if all( + [ + settings.FEDERATEDCODE_VULNERABILITIES_REPO, + settings.FEDERATEDCODE_GIT_SERVICE_TOKEN, + settings.FEDERATEDCODE_GIT_SERVICE_EMAIL, + settings.FEDERATEDCODE_GIT_SERVICE_NAME, + ] + ): + return True + return False + + +def create_federatedcode_working_dir(): + """Create temporary working dir for cloning federatedcode repositories.""" + return Path(tempfile.mkdtemp()) + + +def is_available(): + """Return True if the configured Git repo is available.""" + if not is_configured(): + return False + + return url_exists(settings.FEDERATEDCODE_VULNERABILITIES_REPO) + + +def check_federatedcode_configured_and_available(logger): + """ + Check if the criteria for pushing the results to FederatedCode + is satisfied. + + Criteria: + - FederatedCode is configured and available. + """ + if not is_configured(): + raise Exception("FederatedCode is not configured.") + + if not is_available(): + raise Exception("FederatedCode Git account is not available.") + + logger("Federatedcode repositories are configured and available.") + + +def clone_repository(repo_url, clone_path, logger, shallow_clone=True): + """Clone repository to clone_path.""" + logger(f"Cloning repository {repo_url}") + + authenticated_repo_url = repo_url.replace( + "https://", + f"https://{settings.FEDERATEDCODE_GIT_SERVICE_TOKEN}@", + ) + clone_args = { + "url": authenticated_repo_url, + "to_path": clone_path, + } + if shallow_clone: + clone_args["depth"] = 1 + + repo = Repo.clone_from(**clone_args) + repo.config_writer(config_level="repository").set_value( + "user", "name", settings.FEDERATEDCODE_GIT_SERVICE_NAME + ).release() + repo.config_writer(config_level="repository").set_value( + "user", "email", settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + ).release() + + return repo + + +def get_github_org(url): + """Return org username from GitHub account URL.""" + github_account_url = urlparse(url) + path_after_domain = github_account_url.path.lstrip("/") + org_name = path_after_domain.split("/")[0] + return org_name + + +def push_changes(repo, remote_name="origin", branch_name=""): + """Push changes to remote repository.""" + if not branch_name: + branch_name = repo.active_branch.name + repo.git.push(remote_name, branch_name, "--no-verify") + + +def commit_and_push_changes( + repo, + files_to_commit, + commit_message, + logger, + remote_name="origin", +): + """ + Commit and push changes to remote repository. + Returns True if changes are successfully pushed, False otherwise. + """ + try: + commit_changes(repo, files_to_commit, commit_message) + push_changes(repo, remote_name) + except GitCommandError as e: + if "nothing to commit" in e.stdout.lower(): + logger("Nothing to commit, working tree clean.") + else: + logger(f"Error while committing change: {e}") + return False + return True + + +def commit_changes(repo, files_to_commit, commit_message): + """Commit changes in files to a remote repository.""" + if not files_to_commit: + return + + repo.index.add(files_to_commit) + repo.git.commit( + m=textwrap.dedent(commit_message), + allow_empty=False, + no_verify=True, + ) + + +def commit_message(commit_count, total_commit_count): + """Commit message for pushing Package vulnerability.""" + from vulnerablecode import __version__ as VERSION + + author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME + author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + + tool_name = "pkg:github/aboutcode-org/vulnerablecode" + + return f"""\ + Add new Package vulnerability ({commit_count}/{total_commit_count}) + + Tool: {tool_name}@v{VERSION} + + Signed-off-by: {author_name} <{author_email}> + """ diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 7318e20fb..ae6638b76 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -389,3 +389,13 @@ "DEFAULT_TIMEOUT": env.int("VULNERABLECODE_REDIS_DEFAULT_TIMEOUT", default=3600), } } + + +# FederatedCode integration + +FEDERATEDCODE_VULNERABILITIES_REPO = env.str( + "FEDERATEDCODE_VULNERABILITIES_REPO", default="" +).rstrip("/") +FEDERATEDCODE_GIT_SERVICE_TOKEN = env.str("FEDERATEDCODE_GIT_SERVICE_TOKEN", default="") +FEDERATEDCODE_GIT_SERVICE_NAME = env.str("FEDERATEDCODE_GIT_SERVICE_NAME", default="") +FEDERATEDCODE_GIT_SERVICE_EMAIL = env.str("FEDERATEDCODE_GIT_SERVICE_EMAIL", default="") From bbe43cb540b9102f9d1507ec776400c283e2e832 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:23:31 +0530 Subject: [PATCH 04/26] Add export registry Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/exporters/__init__.py | 16 ++++++++++++++++ ...rabilities.py => federate_vulnerabilities.py} | 0 2 files changed, 16 insertions(+) create mode 100644 vulnerabilities/pipelines/exporters/__init__.py rename vulnerabilities/pipelines/exporters/{federate_package_vulnerabilities.py => federate_vulnerabilities.py} (100%) diff --git a/vulnerabilities/pipelines/exporters/__init__.py b/vulnerabilities/pipelines/exporters/__init__.py new file mode 100644 index 000000000..d158a8967 --- /dev/null +++ b/vulnerabilities/pipelines/exporters/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from vulnerabilities.pipelines.exporters import federate_vulnerabilities +from vulnerabilities.utils import create_registry + +EXPORTERS_REGISTRY = create_registry( + [ + federate_vulnerabilities.FederatePackageVulnerabilities, + ] +) diff --git a/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py similarity index 100% rename from vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py rename to vulnerabilities/pipelines/exporters/federate_vulnerabilities.py From 78421411a8bce4eb1f1ef89f2338210fb425b233 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:41:56 +0530 Subject: [PATCH 05/26] Export package and advisory in two separate steps Signed-off-by: Keshav Priyadarshi --- .../exporters/federate_vulnerabilities.py | 173 ++++++++++++------ vulnerabilities/pipes/federatedcode.py | 4 +- 2 files changed, 121 insertions(+), 56 deletions(-) diff --git a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py index 3bc3dcbaf..aec408d2b 100644 --- a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py +++ b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py @@ -15,8 +15,11 @@ import saneyaml from aboutcode.pipeline import LoopProgress from django.conf import settings +from django.db.models import Prefetch from aboutcode.federated import DataFederation +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes import federatedcode @@ -25,7 +28,7 @@ class FederatePackageVulnerabilities(VulnerableCodePipeline): """Export package vulnerabilities and advisory to FederatedCode.""" - pipeline_id = "federate_package_vulnerabilities_v2" + pipeline_id = "federate_vulnerabilities_v2" @classmethod def steps(cls): @@ -34,7 +37,8 @@ def steps(cls): cls.create_federatedcode_working_dir, cls.fetch_federation_config, cls.clone_vulnerabilities_repo, - cls.publish_vulnerabilities, + cls.publish_package_vulnerabilities, + cls.publish_advisories, cls.delete_working_dir, ) @@ -61,13 +65,13 @@ def clone_vulnerabilities_repo(self): logger=self.log, ) - def publish_vulnerabilities(self): - """Publish package vulnerabilities and advisory to FederatedCode""" + def publish_package_vulnerabilities(self): + """Publish package vulnerabilities to FederatedCode""" repo_path = Path(self.repo.working_dir) commit_count = 1 batch_size = 2000 + chunk_size = 1000 files_to_commit = set() - exported_avids = set() distinct_packages_count = ( PackageV2.objects.values("type", "namespace", "name") @@ -76,44 +80,19 @@ def publish_vulnerabilities(self): ) package_qs = package_prefetched_qs() grouped_packages = itertools.groupby( - package_qs.iterator(chunk_size=2000), + package_qs.iterator(chunk_size=chunk_size), key=attrgetter("type", "namespace", "name"), ) self.log(f"Exporting vulnerabilities for {distinct_packages_count} packages.") progress = LoopProgress( total_iterations=distinct_packages_count, - progress_step=1, + progress_step=5, logger=self.log, ) for _, packages in progress.iter(grouped_packages): - package_urls = [] - package_vulnerabilities = [] - for package in packages: - purl = package.package_url - package_urls.append(purl) - package_vulnerabilities.append(serialize_package_vulnerability(package)) - - impacts = itertools.chain( - package.affected_in_impacts.all(), - package.fixed_in_impacts.all(), - ) - for impact in impacts: - adv = impact.advisory - avid = adv.avid - if avid in exported_avids: - continue - - exported_avids.add(avid) - advisory = serialize_advisory(adv) - adv_file = f"vulnerabilities/{avid}.yml" - write_file( - repo_path=repo_path, - file_path=adv_file, - data=advisory, - ) - files_to_commit.add(adv_file) - + package_urls, package_vulnerabilities = get_package_vulnerabilities(packages) + purl = package_urls[0] package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl=purl) package_vulnerability_path = datafile_path.replace("/purls.yml", "/vulnerabilities.yml") package_vulnerability_path = f"packages/{package_repo}/{package_vulnerability_path}" @@ -135,7 +114,7 @@ def publish_vulnerabilities(self): if len(files_to_commit) > batch_size: if federatedcode.commit_and_push_changes( - commit_message=self.commit_message(commit_count), + commit_message=self.commit_message("package vulnerabilities", commit_count), repo=self.repo, files_to_commit=files_to_commit, logger=self.log, @@ -145,15 +124,67 @@ def publish_vulnerabilities(self): if files_to_commit: federatedcode.commit_and_push_changes( - commit_message=self.commit_message(commit_count, commit_count), + commit_message=self.commit_message( + "package vulnerabilities", + commit_count, + commit_count, + ), repo=self.repo, files_to_commit=files_to_commit, logger=self.log, ) - self.log( - f"Federated {distinct_packages_count} package and {len(exported_avids)} vulnerabilities." + self.log(f"Federated {distinct_packages_count} package vulnerabilities.") + + def publish_advisories(self): + """Publish advisory to FederatedCode""" + repo_path = Path(self.repo.working_dir) + commit_count = 1 + batch_size = 2000 + chunk_size = 1000 + files_to_commit = set() + advisory_qs = advisory_prefetched_qs() + advisory_count = advisory_qs.count() + + self.log(f"Exporting vulnerabilities for {advisory_count} advisory.") + progress = LoopProgress( + total_iterations=advisory_count, + progress_step=5, + logger=self.log, ) + for advisory in progress.iter(advisory_qs.iterator(chunk_size=chunk_size)): + advisory_data = serialize_advisory(advisory) + adv_file = f"vulnerabilities/{advisory.avid}.yml" + write_file( + repo_path=repo_path, + file_path=adv_file, + data=advisory_data, + ) + files_to_commit.add(adv_file) + + if len(files_to_commit) > batch_size: + if federatedcode.commit_and_push_changes( + commit_message=self.commit_message("advisories", commit_count), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ): + commit_count += 1 + files_to_commit.clear() + + if files_to_commit: + federatedcode.commit_and_push_changes( + commit_message=self.commit_message( + "advisories", + commit_count, + commit_count, + ), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ) + + self.log(f"Successfully federated {advisory_count} vulnerabilities.") def delete_working_dir(self): """Remove temporary working dir.""" @@ -163,33 +194,67 @@ def delete_working_dir(self): def on_failure(self): self.delete_working_dir() - def commit_message(self, commit_count, total_commit_count="many"): + def commit_message( + self, + item_type, + commit_count, + total_commit_count="many", + ): """Commit message for pushing Package vulnerability.""" return federatedcode.commit_message( + item_type=item_type, commit_count=commit_count, total_commit_count=total_commit_count, ) def package_prefetched_qs(): - return PackageV2.objects.order_by("type", "namespace", "name", "version").prefetch_related( - "affected_in_impacts", - "affected_in_impacts__advisory", - "affected_in_impacts__advisory__impacted_packages", - "affected_in_impacts__advisory__aliases", - "affected_in_impacts__advisory__references", - "affected_in_impacts__advisory__severities", - "affected_in_impacts__advisory__weaknesses", - "fixed_in_impacts", - "fixed_in_impacts__advisory", - "fixed_in_impacts__advisory__impacted_packages", - "fixed_in_impacts__advisory__aliases", - "fixed_in_impacts__advisory__references", - "fixed_in_impacts__advisory__severities", - "fixed_in_impacts__advisory__weaknesses", + return ( + PackageV2.objects.order_by("type", "namespace", "name", "version") + .only("id", "package_url", "type", "namespace", "name", "version") + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related( + Prefetch( + "advisory", + queryset=AdvisoryV2.objects.only("id", "avid"), + ) + ), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related( + Prefetch( + "advisory", + queryset=AdvisoryV2.objects.only("id", "avid"), + ) + ), + ), + ) ) +def advisory_prefetched_qs(): + return AdvisoryV2.objects.prefetch_related( + "impacted_packages", + "aliases", + "references", + "severities", + "weaknesses", + ) + + +def get_package_vulnerabilities(packages): + """Return list of PURLs and serialized package vulnerability""" + package_urls = [] + package_vulnerabilities = [] + for package in packages: + package_urls.append(package.package_url) + package_vulnerabilities.append(serialize_package_vulnerability(package)) + return package_urls, package_vulnerabilities + + def serialize_package_vulnerability(package): affected_by_vulnerabilities = [ impact.advisory.avid for impact in package.affected_in_impacts.all() diff --git a/vulnerabilities/pipes/federatedcode.py b/vulnerabilities/pipes/federatedcode.py index 604c79237..560519c8d 100644 --- a/vulnerabilities/pipes/federatedcode.py +++ b/vulnerabilities/pipes/federatedcode.py @@ -157,7 +157,7 @@ def commit_changes(repo, files_to_commit, commit_message): ) -def commit_message(commit_count, total_commit_count): +def commit_message(item_type, commit_count, total_commit_count): """Commit message for pushing Package vulnerability.""" from vulnerablecode import __version__ as VERSION @@ -167,7 +167,7 @@ def commit_message(commit_count, total_commit_count): tool_name = "pkg:github/aboutcode-org/vulnerablecode" return f"""\ - Add new Package vulnerability ({commit_count}/{total_commit_count}) + Add new {item_type} ({commit_count}/{total_commit_count}) Tool: {tool_name}@v{VERSION} From eacb31586f202a09ca253541ea5a99942ae68351 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:49:06 +0530 Subject: [PATCH 06/26] Add tests for v2 vulnerability exporter pipeline Signed-off-by: Keshav Priyadarshi --- .../test_federate_vulnerabilities.py | 103 ++++++++++++++++++ .../ADV-123-expected.yml | 16 +++ .../purls-expected.yml | 4 + .../vulnerabilities-expected.yml | 16 +++ 4 files changed, 139 insertions(+) create mode 100644 vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml diff --git a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py new file mode 100644 index 000000000..8572d55dd --- /dev/null +++ b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py @@ -0,0 +1,103 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import tempfile +from datetime import datetime +from datetime import timedelta +from pathlib import Path +from unittest.mock import patch + +from django.test import TestCase +from git import Repo +from packageurl import PackageURL +from univers.version_range import VersionRange + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.pipelines import insert_advisory_v2 +from vulnerabilities.pipelines.exporters.federate_vulnerabilities import ( + FederatePackageVulnerabilities, +) +from vulnerabilities.tests import util_tests +from vulnerabilities.tests.pipelines import TestLogger + +TEST_DATA = ( + Path(__file__).parent.parent.parent / "test_data" / "exporters" / "federate_vulnerabilities" +) + + +class TestFederatePackageVulnerabilities(TestCase): + def setUp(self): + self.logger = TestLogger() + + advisory = AdvisoryDataV2( + summary="Test advisory", + aliases=["CVE-2025-0001"], + references=[], + severities=[], + weaknesses=[], + affected_packages=[ + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=1.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/1.2.4"), + introduced_by_commit_patches=[], + fixed_by_commit_patches=[], + ), + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"), + introduced_by_commit_patches=[], + fixed_by_commit_patches=[], + ), + ], + patches=[], + advisory_id="ADV-123", + date_published=datetime.now() - timedelta(days=10), + url="https://example.com/advisory/1", + ) + insert_advisory_v2( + advisory=advisory, + pipeline_id="test_pipeline_v2", + ) + + @patch( + "vulnerabilities.pipelines.exporters.federate_vulnerabilities.FederatePackageVulnerabilities.clone_vulnerabilities_repo" + ) + @patch("vulnerabilities.pipes.federatedcode.commit_and_push_changes") + @patch("vulnerabilities.pipes.federatedcode.check_federatedcode_configured_and_available") + def test_vulnerabilities_federation_v2(self, mock_check_fed, mock_commit, mock_clone): + mock_check_fed.return_value = None + mock_commit.return_value = None + mock_clone.__name__ = "clone_vulnerabilities_repo" + + working_dir = Path(tempfile.mkdtemp()) + print(working_dir) + + pipeline = FederatePackageVulnerabilities() + pipeline.repo = Repo.init(working_dir) + pipeline.log = self.logger.write + pipeline.execute() + print(self.logger.getvalue()) + + result_purl_yml = next(working_dir.rglob("purls.yml")) + result_vulnerabilities_yml = next(working_dir.rglob("vulnerabilities.yml")) + result_advisory_yml = next(working_dir.rglob("ADV-123.yml")) + + expected_purl_yml = TEST_DATA / "purls-expected.yml" + expected_vulnerabilities_yml = TEST_DATA / "vulnerabilities-expected.yml" + expected_advisory_yml = TEST_DATA / "ADV-123-expected.yml" + + util_tests.check_results_and_expected_files(result_purl_yml, expected_purl_yml) + util_tests.check_results_and_expected_files( + result_vulnerabilities_yml, expected_vulnerabilities_yml + ) + util_tests.check_results_and_expected_files(result_advisory_yml, expected_advisory_yml) diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml new file mode 100644 index 000000000..2411b65ef --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml @@ -0,0 +1,16 @@ +advisory_id: ADV-123 +datasource_id: test_pipeline_v2/ADV-123 +datasource_url: https://example.com/advisory/1 +aliases: + - CVE-2025-0001 +summary: Test advisory +impacted_packages: + - purl: pkg:npm/foobar + affected_versions: vers:npm/<=1.2.3 + fixed_versions: vers:npm/1.2.4 + - purl: pkg:npm/foobar + affected_versions: vers:npm/<=3.2.3 + fixed_versions: vers:npm/3.2.4 +severities: [] +weaknesses: [] +references: [] diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml new file mode 100644 index 000000000..6721cc802 --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml @@ -0,0 +1,4 @@ +- pkg:npm/foobar@1.2.3 +- pkg:npm/foobar@1.2.4 +- pkg:npm/foobar@3.2.3 +- pkg:npm/foobar@3.2.4 diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml new file mode 100644 index 000000000..fb0677d34 --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml @@ -0,0 +1,16 @@ +- purl: pkg:npm/foobar@1.2.3 + affected_by_vulnerabilities: + - test_pipeline_v2/ADV-123 + fixing_vulnerabilities: [] +- purl: pkg:npm/foobar@1.2.4 + affected_by_vulnerabilities: [] + fixing_vulnerabilities: + - test_pipeline_v2/ADV-123 +- purl: pkg:npm/foobar@3.2.3 + affected_by_vulnerabilities: + - test_pipeline_v2/ADV-123 + fixing_vulnerabilities: [] +- purl: pkg:npm/foobar@3.2.4 + affected_by_vulnerabilities: [] + fixing_vulnerabilities: + - test_pipeline_v2/ADV-123 From b9b9a44b1d7fdbae336de15bf0ae533363ba6958 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:49:29 +0530 Subject: [PATCH 07/26] Enable scheduling for exporter pipelines Signed-off-by: Keshav Priyadarshi --- vulnerabilities/schedules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/schedules.py b/vulnerabilities/schedules.py index 8ae3bbb93..e6443e5ab 100644 --- a/vulnerabilities/schedules.py +++ b/vulnerabilities/schedules.py @@ -88,8 +88,9 @@ def update_pipeline_schedule(): from vulnerabilities.importers import IMPORTERS_REGISTRY from vulnerabilities.improvers import IMPROVERS_REGISTRY from vulnerabilities.models import PipelineSchedule + from vulnerabilities.pipelines.exporters import EXPORTERS_REGISTRY - pipelines = IMPORTERS_REGISTRY | IMPROVERS_REGISTRY + pipelines = IMPORTERS_REGISTRY | IMPROVERS_REGISTRY | EXPORTERS_REGISTRY PipelineSchedule.objects.exclude(pipeline_id__in=pipelines.keys()).delete() for id, pipeline_class in pipelines.items(): From c60eeaf96972391edca2c339633b8c00ad4d8b68 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 17 Feb 2026 00:35:13 +0530 Subject: [PATCH 08/26] Store package advisory relation in advisories.yml Signed-off-by: Keshav Priyadarshi --- aboutcode/federated/__init__.py | 8 ++++---- .../all-presets/foo/aboutcode-federated-config.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aboutcode/federated/__init__.py b/aboutcode/federated/__init__.py index 321013710..ff3f07e84 100644 --- a/aboutcode/federated/__init__.py +++ b/aboutcode/federated/__init__.py @@ -1028,7 +1028,7 @@ def large_size_configs(cls): "mlflow": 16, "pub": 16, "rpm": 16, - # Small Ecosystem all use the defaul + # Small Ecosystem all use the default "default": 1, } return [ @@ -1069,7 +1069,7 @@ def medium_size_configs(cls): "mlflow": 8, "pub": 8, "rpm": 8, - # Small Ecosystem all use the defaul + # Small Ecosystem all use the default "default": 1, } return [ @@ -1110,7 +1110,7 @@ def small_size_configs(cls): "mlflow": 4, "pub": 4, "rpm": 4, - # Small Ecosystem all use the defaul + # Small Ecosystem all use the default "default": 1, } return [ @@ -1181,7 +1181,7 @@ def cluster_preset(): DataCluster( data_kind="security_advisories", description="VulnerableCode security advisories for each package version.", - datafile_path_template="{/namespace}/{name}/{version}/advisories.json", + datafile_path_template="{/namespace}/{name}/{version}/advisories.yml", purl_type_configs=[PurlTypeConfig.default_config()], data_schema_url="", documentation_url="", diff --git a/aboutcode/federated/tests/test_data/all-presets/foo/aboutcode-federated-config.yml b/aboutcode/federated/tests/test_data/all-presets/foo/aboutcode-federated-config.yml index f7a7b89c6..ca4204bb7 100644 --- a/aboutcode/federated/tests/test_data/all-presets/foo/aboutcode-federated-config.yml +++ b/aboutcode/federated/tests/test_data/all-presets/foo/aboutcode-federated-config.yml @@ -933,7 +933,7 @@ data_clusters: data_license: CC-BY-4.0 data_maintainers: [] - data_kind: security_advisories - datafile_path_template: '{/namespace}/{name}/{version}/advisories.json' + datafile_path_template: '{/namespace}/{name}/{version}/advisories.yml' purl_type_configs: - purl_type: default number_of_repos: 1 From 15a84926f645b254fb0a2e7459317ebf70fcaec7 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 17 Feb 2026 00:36:40 +0530 Subject: [PATCH 09/26] Support export pipeline in pipeline schedules Signed-off-by: Keshav Priyadarshi --- vulnerabilities/models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index c102a697a..a83db0ad6 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2344,13 +2344,14 @@ def save(self, *args, **kwargs): @property def pipeline_class(self): """Return the pipeline class.""" + from vulnerabilities.importers import IMPORTERS_REGISTRY from vulnerabilities.improvers import IMPROVERS_REGISTRY + from vulnerabilities.pipelines.exporters import EXPORTERS_REGISTRY + + pipeline_registry = IMPORTERS_REGISTRY | IMPROVERS_REGISTRY | EXPORTERS_REGISTRY - if self.pipeline_id in IMPROVERS_REGISTRY: - return IMPROVERS_REGISTRY.get(self.pipeline_id) - if self.pipeline_id in IMPORTERS_REGISTRY: - return IMPORTERS_REGISTRY.get(self.pipeline_id) + return pipeline_registry[self.pipeline_id] @property def description(self): From 365f6ce91d8e35eb0230f96a057ba9df52c67dc0 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 17 Feb 2026 14:36:42 +0530 Subject: [PATCH 10/26] Store advisories in security_advisories cluster Signed-off-by: Keshav Priyadarshi --- .../exporters/federate_vulnerabilities.py | 107 ++++++++---------- .../test_federate_vulnerabilities.py | 56 +++++---- .../1.2.4/advisories-expected.yml | 5 + ...-123-expected.yml => ADV-001-expected.yml} | 7 +- .../ADV-002-expected.yml | 13 +++ .../purls-expected.yml | 4 - .../vulnerabilities-expected.yml | 16 --- 7 files changed, 100 insertions(+), 108 deletions(-) create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/1.2.4/advisories-expected.yml rename vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/{ADV-123-expected.yml => ADV-001-expected.yml} (61%) create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml delete mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml delete mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml diff --git a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py index aec408d2b..27c6546ff 100644 --- a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py +++ b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py @@ -36,8 +36,8 @@ def steps(cls): cls.check_federatedcode_eligibility, cls.create_federatedcode_working_dir, cls.fetch_federation_config, - cls.clone_vulnerabilities_repo, - cls.publish_package_vulnerabilities, + cls.clone_federation_repository, + cls.publish_package_related_advisories, cls.publish_advisories, cls.delete_working_dir, ) @@ -56,54 +56,44 @@ def fetch_federation_config(self): name="aboutcode-data", remote_root_url="https://github.com/aboutcode-data", ) - self.data_cluster = data_federation.get_cluster("purls") + self.data_cluster = data_federation.get_cluster("security_advisories") - def clone_vulnerabilities_repo(self): + def clone_federation_repository(self): self.repo = federatedcode.clone_repository( repo_url=settings.FEDERATEDCODE_VULNERABILITIES_REPO, - clone_path=self.working_path / "vulnerabilities-data", + clone_path=self.working_path / "advisories-data", logger=self.log, ) - def publish_package_vulnerabilities(self): - """Publish package vulnerabilities to FederatedCode""" + def publish_package_related_advisories(self): + """Publish package advisories relations to FederatedCode""" repo_path = Path(self.repo.working_dir) commit_count = 1 batch_size = 2000 - chunk_size = 1000 + chunk_size = 500 files_to_commit = set() distinct_packages_count = ( - PackageV2.objects.values("type", "namespace", "name") - .distinct("type", "namespace", "name") + PackageV2.objects.values("type", "namespace", "name", "version") + .distinct("type", "namespace", "name", "version") .count() ) package_qs = package_prefetched_qs() grouped_packages = itertools.groupby( package_qs.iterator(chunk_size=chunk_size), - key=attrgetter("type", "namespace", "name"), + key=attrgetter("type", "namespace", "name", "version"), ) - self.log(f"Exporting vulnerabilities for {distinct_packages_count} packages.") + self.log(f"Exporting advisory relation for {distinct_packages_count} packages.") progress = LoopProgress( total_iterations=distinct_packages_count, progress_step=5, logger=self.log, ) for _, packages in progress.iter(grouped_packages): - package_urls, package_vulnerabilities = get_package_vulnerabilities(packages) - purl = package_urls[0] - package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl=purl) - package_vulnerability_path = datafile_path.replace("/purls.yml", "/vulnerabilities.yml") - package_vulnerability_path = f"packages/{package_repo}/{package_vulnerability_path}" - package_path = f"packages/{package_repo}/{datafile_path}" - - write_file( - repo_path=repo_path, - file_path=package_path, - data=package_urls, - ) - files_to_commit.add(package_path) + purl, package_vulnerabilities = get_package_related_advisory(packages) + package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl) + package_vulnerability_path = f"packages/{package_repo}/{datafile_path}" write_file( repo_path=repo_path, @@ -114,7 +104,7 @@ def publish_package_vulnerabilities(self): if len(files_to_commit) > batch_size: if federatedcode.commit_and_push_changes( - commit_message=self.commit_message("package vulnerabilities", commit_count), + commit_message=self.commit_message("package advisory relations", commit_count), repo=self.repo, files_to_commit=files_to_commit, logger=self.log, @@ -125,7 +115,7 @@ def publish_package_vulnerabilities(self): if files_to_commit: federatedcode.commit_and_push_changes( commit_message=self.commit_message( - "package vulnerabilities", + "package advisory relations", commit_count, commit_count, ), @@ -134,7 +124,7 @@ def publish_package_vulnerabilities(self): logger=self.log, ) - self.log(f"Federated {distinct_packages_count} package vulnerabilities.") + self.log(f"Federated {distinct_packages_count} package advisories.") def publish_advisories(self): """Publish advisory to FederatedCode""" @@ -146,7 +136,7 @@ def publish_advisories(self): advisory_qs = advisory_prefetched_qs() advisory_count = advisory_qs.count() - self.log(f"Exporting vulnerabilities for {advisory_count} advisory.") + self.log(f"Exporting {advisory_count} advisory.") progress = LoopProgress( total_iterations=advisory_count, progress_step=5, @@ -154,7 +144,7 @@ def publish_advisories(self): ) for advisory in progress.iter(advisory_qs.iterator(chunk_size=chunk_size)): advisory_data = serialize_advisory(advisory) - adv_file = f"vulnerabilities/{advisory.avid}.yml" + adv_file = f"advisories/{advisory.avid}.yml" write_file( repo_path=repo_path, file_path=adv_file, @@ -184,7 +174,7 @@ def publish_advisories(self): logger=self.log, ) - self.log(f"Successfully federated {advisory_count} vulnerabilities.") + self.log(f"Successfully federated {advisory_count} advisories.") def delete_working_dir(self): """Remove temporary working dir.""" @@ -200,7 +190,7 @@ def commit_message( commit_count, total_commit_count="many", ): - """Commit message for pushing Package vulnerability.""" + """Commit message for pushing package vulnerability.""" return federatedcode.commit_message( item_type=item_type, commit_count=commit_count, @@ -211,23 +201,23 @@ def commit_message( def package_prefetched_qs(): return ( PackageV2.objects.order_by("type", "namespace", "name", "version") - .only("id", "package_url", "type", "namespace", "name", "version") + .only("package_url", "type", "namespace", "name", "version") .prefetch_related( Prefetch( "affected_in_impacts", - queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related( + queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( Prefetch( "advisory", - queryset=AdvisoryV2.objects.only("id", "avid"), + queryset=AdvisoryV2.objects.only("avid"), ) ), ), Prefetch( "fixed_in_impacts", - queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related( + queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( Prefetch( "advisory", - queryset=AdvisoryV2.objects.only("id", "avid"), + queryset=AdvisoryV2.objects.only("avid"), ) ), ), @@ -235,6 +225,24 @@ def package_prefetched_qs(): ) +def get_package_related_advisory(packages): + package_vulnerabilities = [] + for package in packages: + affected_by_vulnerabilities = [ + impact.advisory.avid for impact in package.affected_in_impacts.all() + ] + fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()] + + package_vulnerability = { + "purl": package.package_url, + "affected_by_advisories": sorted(affected_by_vulnerabilities), + "fixing_advisories": sorted(fixing_vulnerabilities), + } + package_vulnerabilities.append(package_vulnerability) + + return package.package_url, package_vulnerabilities + + def advisory_prefetched_qs(): return AdvisoryV2.objects.prefetch_related( "impacted_packages", @@ -245,29 +253,6 @@ def advisory_prefetched_qs(): ) -def get_package_vulnerabilities(packages): - """Return list of PURLs and serialized package vulnerability""" - package_urls = [] - package_vulnerabilities = [] - for package in packages: - package_urls.append(package.package_url) - package_vulnerabilities.append(serialize_package_vulnerability(package)) - return package_urls, package_vulnerabilities - - -def serialize_package_vulnerability(package): - affected_by_vulnerabilities = [ - impact.advisory.avid for impact in package.affected_in_impacts.all() - ] - fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()] - - return { - "purl": package.package_url, - "affected_by_vulnerabilities": affected_by_vulnerabilities, - "fixing_vulnerabilities": fixing_vulnerabilities, - } - - def serialize_severity(sev): return { "score": sev.value, @@ -288,7 +273,7 @@ def serialize_references(reference): def serialize_advisory(advisory): """Return a plain data mapping serialized from advisory object.""" - aliases = [a.alias for a in advisory.aliases.all()] + aliases = sorted([a.alias for a in advisory.aliases.all()]) severities = [serialize_severity(sev) for sev in advisory.severities.all()] weaknesses = [wkns.cwe for wkns in advisory.weaknesses.all()] references = [serialize_references(ref) for ref in advisory.references.all()] diff --git a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py index 8572d55dd..0d6fa32a5 100644 --- a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py +++ b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py @@ -37,7 +37,7 @@ class TestFederatePackageVulnerabilities(TestCase): def setUp(self): self.logger = TestLogger() - advisory = AdvisoryDataV2( + advisory1 = AdvisoryDataV2( summary="Test advisory", aliases=["CVE-2025-0001"], references=[], @@ -51,53 +51,65 @@ def setUp(self): introduced_by_commit_patches=[], fixed_by_commit_patches=[], ), + ], + patches=[], + advisory_id="ADV-001", + date_published=datetime.now() - timedelta(days=10), + url="https://example.com/advisory/1", + ) + advisory2 = AdvisoryDataV2( + summary="Test advisory2", + aliases=["CVE-2025-0002"], + references=[], + severities=[], + weaknesses=[], + affected_packages=[ AffectedPackageV2( package=PackageURL.from_string("pkg:npm/foobar"), - affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"), - fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"), + affected_version_range=VersionRange.from_string("vers:npm/>=1.2.4"), + fixed_version_range=VersionRange.from_string("vers:npm/2.0.0"), introduced_by_commit_patches=[], fixed_by_commit_patches=[], ), ], patches=[], - advisory_id="ADV-123", + advisory_id="ADV-002", date_published=datetime.now() - timedelta(days=10), - url="https://example.com/advisory/1", + url="https://example.com/advisory/2", ) insert_advisory_v2( - advisory=advisory, + advisory=advisory1, + pipeline_id="test_pipeline_v2", + ) + insert_advisory_v2( + advisory=advisory2, pipeline_id="test_pipeline_v2", ) @patch( - "vulnerabilities.pipelines.exporters.federate_vulnerabilities.FederatePackageVulnerabilities.clone_vulnerabilities_repo" + "vulnerabilities.pipelines.exporters.federate_vulnerabilities.FederatePackageVulnerabilities.clone_federation_repository" ) @patch("vulnerabilities.pipes.federatedcode.commit_and_push_changes") @patch("vulnerabilities.pipes.federatedcode.check_federatedcode_configured_and_available") def test_vulnerabilities_federation_v2(self, mock_check_fed, mock_commit, mock_clone): mock_check_fed.return_value = None mock_commit.return_value = None - mock_clone.__name__ = "clone_vulnerabilities_repo" + mock_clone.__name__ = "clone_federation_repository" working_dir = Path(tempfile.mkdtemp()) - print(working_dir) - pipeline = FederatePackageVulnerabilities() pipeline.repo = Repo.init(working_dir) pipeline.log = self.logger.write pipeline.execute() - print(self.logger.getvalue()) - result_purl_yml = next(working_dir.rglob("purls.yml")) - result_vulnerabilities_yml = next(working_dir.rglob("vulnerabilities.yml")) - result_advisory_yml = next(working_dir.rglob("ADV-123.yml")) + result_advisories_yml = next(working_dir.rglob("1.2.4/advisories.yml")) + result_advisory1_yml = next(working_dir.rglob("ADV-001.yml")) + result_advisory2_yml = next(working_dir.rglob("ADV-002.yml")) - expected_purl_yml = TEST_DATA / "purls-expected.yml" - expected_vulnerabilities_yml = TEST_DATA / "vulnerabilities-expected.yml" - expected_advisory_yml = TEST_DATA / "ADV-123-expected.yml" + expected_advisories_yml = TEST_DATA / "1.2.4" / "advisories-expected.yml" + expected_advisory1_yml = TEST_DATA / "ADV-001-expected.yml" + expected_advisory2_yml = TEST_DATA / "ADV-002-expected.yml" - util_tests.check_results_and_expected_files(result_purl_yml, expected_purl_yml) - util_tests.check_results_and_expected_files( - result_vulnerabilities_yml, expected_vulnerabilities_yml - ) - util_tests.check_results_and_expected_files(result_advisory_yml, expected_advisory_yml) + util_tests.check_results_and_expected_files(result_advisories_yml, expected_advisories_yml) + util_tests.check_results_and_expected_files(result_advisory1_yml, expected_advisory1_yml) + util_tests.check_results_and_expected_files(result_advisory2_yml, expected_advisory2_yml) diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/1.2.4/advisories-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/1.2.4/advisories-expected.yml new file mode 100644 index 000000000..eb0c3737d --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/1.2.4/advisories-expected.yml @@ -0,0 +1,5 @@ +- purl: pkg:npm/foobar@1.2.4 + affected_by_advisories: + - test_pipeline_v2/ADV-002 + fixing_advisories: + - test_pipeline_v2/ADV-001 diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml similarity index 61% rename from vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml rename to vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml index 2411b65ef..de4faff4f 100644 --- a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml @@ -1,5 +1,5 @@ -advisory_id: ADV-123 -datasource_id: test_pipeline_v2/ADV-123 +advisory_id: ADV-001 +datasource_id: test_pipeline_v2/ADV-001 datasource_url: https://example.com/advisory/1 aliases: - CVE-2025-0001 @@ -8,9 +8,6 @@ impacted_packages: - purl: pkg:npm/foobar affected_versions: vers:npm/<=1.2.3 fixed_versions: vers:npm/1.2.4 - - purl: pkg:npm/foobar - affected_versions: vers:npm/<=3.2.3 - fixed_versions: vers:npm/3.2.4 severities: [] weaknesses: [] references: [] diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml new file mode 100644 index 000000000..1391dfa01 --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml @@ -0,0 +1,13 @@ +advisory_id: ADV-002 +datasource_id: test_pipeline_v2/ADV-002 +datasource_url: https://example.com/advisory/2 +aliases: + - CVE-2025-0002 +summary: Test advisory2 +impacted_packages: + - purl: pkg:npm/foobar + affected_versions: vers:npm/>=1.2.4 + fixed_versions: vers:npm/2.0.0 +severities: [] +weaknesses: [] +references: [] diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml deleted file mode 100644 index 6721cc802..000000000 --- a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml +++ /dev/null @@ -1,4 +0,0 @@ -- pkg:npm/foobar@1.2.3 -- pkg:npm/foobar@1.2.4 -- pkg:npm/foobar@3.2.3 -- pkg:npm/foobar@3.2.4 diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml deleted file mode 100644 index fb0677d34..000000000 --- a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml +++ /dev/null @@ -1,16 +0,0 @@ -- purl: pkg:npm/foobar@1.2.3 - affected_by_vulnerabilities: - - test_pipeline_v2/ADV-123 - fixing_vulnerabilities: [] -- purl: pkg:npm/foobar@1.2.4 - affected_by_vulnerabilities: [] - fixing_vulnerabilities: - - test_pipeline_v2/ADV-123 -- purl: pkg:npm/foobar@3.2.3 - affected_by_vulnerabilities: - - test_pipeline_v2/ADV-123 - fixing_vulnerabilities: [] -- purl: pkg:npm/foobar@3.2.4 - affected_by_vulnerabilities: [] - fixing_vulnerabilities: - - test_pipeline_v2/ADV-123 From 092b590a1771532a677052c78c435fae2d711c1d Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 30 Dec 2025 20:44:49 +0200 Subject: [PATCH 11/26] Add initial migration to Gentoo importer v2 Signed-off-by: ziad hany --- vulnerabilities/importers/__init__.py | 2 + .../pipelines/v2_importers/gentoo_importer.py | 175 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 vulnerabilities/pipelines/v2_importers/gentoo_importer.py diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 067f64a7b..7c1cb85df 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -54,6 +54,7 @@ elixir_security_importer as elixir_security_importer_v2, ) from vulnerabilities.pipelines.v2_importers import epss_importer_v2 +from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2 from vulnerabilities.pipelines.v2_importers import fireeye_importer_v2 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2 @@ -108,6 +109,7 @@ project_kb_msr2019_importer_v2.ProjectKBMSR2019Pipeline, ruby_importer_v2.RubyImporterPipeline, epss_importer_v2.EPSSImporterPipeline, + gentoo_importer_v2.GentooImporterPipeline, nginx_importer_v2.NginxImporterPipeline, debian_importer_v2.DebianImporterPipeline, mattermost_importer_v2.MattermostImporterPipeline, diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py new file mode 100644 index 000000000..bab5d12d8 --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -0,0 +1,175 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import re +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Iterable + +from fetchcode.vcs import fetch_via_vcs +from packageurl import PackageURL +from univers.version_constraint import VersionConstraint +from univers.version_range import EbuildVersionRange +from univers.versions import GentooVersion + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 + + +class GentooImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git" + spdx_license_expression = "CC-BY-SA-4.0" + # the license notice is at this url https://anongit.gentoo.org/ says: + # The contents of this document, unless otherwise expressly stated, are licensed + # under the [CC-BY-SA-4.0](https://creativecommons.org/licenses/by-sa/4.0/) license. + license_url = "https://creativecommons.org/licenses/by-sa/4.0/" + pipeline_id = "gentoo_importer_v2" + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + advisory_dir = Path(self.vcs_response.dest_dir) + return sum(1 for _ in advisory_dir.rglob("*.xml")) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + base_path = Path(self.vcs_response.dest_dir) + for file_path in base_path.glob("**/*.xml"): + yield from self.process_file(file_path) + + def process_file(self, file): + cves = [] + summary = "" + vuln_references = [] + xml_root = ET.parse(file).getroot() + id = xml_root.attrib.get("id") + if id: + glsa = "GLSA-" + id + vuln_references = [ + ReferenceV2( + reference_id=glsa, + url=f"https://security.gentoo.org/glsa/{id}", + ) + ] + + for child in xml_root: + if child.tag == "references": + cves = self.cves_from_reference(child) + + if child.tag == "synopsis": + summary = child.text + + if child.tag == "affected": + affected_packages = list(affected_and_safe_purls(child)) + + # It is very inefficient, to create new Advisory for each CVE + # this way, but there seems no alternative. + for cve in cves: + yield AdvisoryData( + advisory_id=cve, + aliases=[cve], + summary=summary, + references=vuln_references, + affected_packages=affected_packages, + url=f"https://security.gentoo.org/glsa/{id}" + if id + else "https://security.gentoo.org/glsa", + ) + + def clean_downloads(self): + if self.vcs_response: + self.log("Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + self.clean_downloads() + + @staticmethod + def cves_from_reference(reference): + cves = [] + for child in reference: + txt = child.text.strip() + match = re.match(r"CVE-\d{4}-\d{4,}", txt) + if match: + cves.append(match.group()) + return cves + + +def affected_and_safe_purls(affected_elem): + constraints = [] + for pkg in affected_elem: + name = pkg.attrib.get("name") + if not name: + continue + pkg_ns, _, pkg_name = name.rpartition("/") + purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns) + safe_versions, affected_versions = get_safe_and_affected_versions(pkg) + + for version in safe_versions: + constraints.append( + VersionConstraint(version=GentooVersion(version), comparator="=").invert() + ) + + for version in affected_versions: + constraints.append(VersionConstraint(version=GentooVersion(version), comparator="=")) + + if not constraints: + continue + + yield AffectedPackageV2( + package=purl, + affected_version_range=EbuildVersionRange(constraints=constraints), + fixed_version_range=None, + ) + + +def get_safe_and_affected_versions(pkg): + # TODO : Revisit why we are skipping some versions in gentoo importer + skip_versions = {"1.3*", "7.3*", "7.4*"} + safe_versions = set() + affected_versions = set() + for info in pkg: + if info.text in skip_versions: + continue + + if info.attrib.get("range"): + if len(info.attrib.get("range")) > 2: + continue + + if info.tag == "unaffected": + # quick hack, to know whether this + # version lies in this range, 'e' stands for + # equal, which is paired with 'greater' or 'less'. + # All possible values of info.attrib['range'] = + # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of + # which ('rle', 'rge', 'rgt') are ignored, because they compare + # 'release' not the 'version'. + if "e" in info.attrib["range"]: + safe_versions.add(info.text) + else: + affected_versions.add(info.text) + + elif info.tag == "vulnerable": + if "e" in info.attrib["range"]: + affected_versions.add(info.text) + else: + safe_versions.add(info.text) + + return safe_versions, affected_versions From 9bb18c2e43d874d9d1e5cfdabe147a995e5f8fde Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 1 Jan 2026 12:38:50 +0200 Subject: [PATCH 12/26] Add a test for gentoo importer v2 Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 69 ++++++++++------- .../v2_importers/test_gentoo_importer_v2.py | 38 +++++++++ .../gentoo_v2/glsa-201709-09-expected.json | 43 +++++++++++ .../test_data/gentoo_v2/glsa-201709-09.xml | 77 +++++++++++++++++++ .../gentoo_v2/glsa-202511-02-expected.json | 67 ++++++++++++++++ .../test_data/gentoo_v2/glsa-202511-02.xml | 73 ++++++++++++++++++ .../gentoo_v2/glsa-202512-01-expected.json | 41 ++++++++++ .../test_data/gentoo_v2/glsa-202512-01.xml | 41 ++++++++++ 8 files changed, 423 insertions(+), 26 deletions(-) create mode 100644 vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py create mode 100644 vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json create mode 100644 vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09.xml create mode 100644 vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json create mode 100644 vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02.xml create mode 100644 vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json create mode 100644 vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01.xml diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index bab5d12d8..61798147c 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -17,11 +17,15 @@ from univers.version_constraint import VersionConstraint from univers.version_range import EbuildVersionRange from univers.versions import GentooVersion +from univers.versions import InvalidVersion from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.management.commands.commit_export import logger from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.severity_systems import GENERIC class GentooImporterPipeline(VulnerableCodeBaseImporterPipelineV2): @@ -57,18 +61,19 @@ def collect_advisories(self) -> Iterable[AdvisoryData]: def process_file(self, file): cves = [] summary = "" - vuln_references = [] xml_root = ET.parse(file).getroot() id = xml_root.attrib.get("id") - if id: - glsa = "GLSA-" + id - vuln_references = [ - ReferenceV2( - reference_id=glsa, - url=f"https://security.gentoo.org/glsa/{id}", - ) - ] + glsa = "GLSA-" + id + + vuln_references = [ + ReferenceV2( + reference_id=glsa, + url=f"https://security.gentoo.org/glsa/{id}", + ) + ] + severities = [] + affected_packages = [] for child in xml_root: if child.tag == "references": cves = self.cves_from_reference(child) @@ -79,19 +84,23 @@ def process_file(self, file): if child.tag == "affected": affected_packages = list(affected_and_safe_purls(child)) - # It is very inefficient, to create new Advisory for each CVE - # this way, but there seems no alternative. - for cve in cves: - yield AdvisoryData( - advisory_id=cve, - aliases=[cve], - summary=summary, - references=vuln_references, - affected_packages=affected_packages, - url=f"https://security.gentoo.org/glsa/{id}" - if id - else "https://security.gentoo.org/glsa", - ) + if child.tag == "impact": + severity_value = child.attrib.get("type") + if severity_value: + severities.append(VulnerabilitySeverity(system=GENERIC, value=severity_value)) + + yield AdvisoryData( + advisory_id=glsa, + aliases=cves, + summary=summary, + references_v2=vuln_references, + severities=severities, + affected_packages=affected_packages, + url=f"https://security.gentoo.org/glsa/{id}" + if id + else "https://security.gentoo.org/glsa", + original_advisory_text=file, + ) def clean_downloads(self): if self.vcs_response: @@ -123,12 +132,20 @@ def affected_and_safe_purls(affected_elem): safe_versions, affected_versions = get_safe_and_affected_versions(pkg) for version in safe_versions: - constraints.append( - VersionConstraint(version=GentooVersion(version), comparator="=").invert() - ) + try: + constraints.append( + VersionConstraint(version=GentooVersion(version), comparator="=").invert() + ) + except InvalidVersion as e: + logger.error(f"InvalidVersion - version: {version} - error:{e}") for version in affected_versions: - constraints.append(VersionConstraint(version=GentooVersion(version), comparator="=")) + try: + constraints.append( + VersionConstraint(version=GentooVersion(version), comparator="=") + ) + except InvalidVersion as e: + logger.error(f"InvalidVersion - version: {version} - error:{e}") if not constraints: continue diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py new file mode 100644 index 000000000..c8206a499 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py @@ -0,0 +1,38 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +from unittest.mock import Mock +from unittest.mock import patch + +import pytest + +from vulnerabilities.pipelines.v2_importers.gentoo_importer import GentooImporterPipeline +from vulnerabilities.tests import util_tests + +TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "gentoo_v2" + +TEST_CVE_FILES = [ + TEST_DATA / "glsa-201709-09.xml", + TEST_DATA / "glsa-202511-02.xml", + TEST_DATA / "glsa-202512-01.xml", +] + + +@pytest.mark.django_db +@pytest.mark.parametrize("xml_file", TEST_CVE_FILES) +def test_gentoo_advisories_per_file(xml_file): + pipeline = GentooImporterPipeline() + pipeline.vcs_response = Mock(dest_dir=TEST_DATA) + + with patch.object(Path, "glob", return_value=[xml_file]): + result = [adv.to_dict() for adv in pipeline.collect_advisories()] + + expected_file = xml_file.with_name(xml_file.stem + "-expected.json") + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json new file mode 100644 index 000000000..d11fedce3 --- /dev/null +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -0,0 +1,43 @@ +[ + { + "advisory_id": "GLSA-201709-09", + "aliases": [ + "CVE-2017-9800" + ], + "summary": "A command injection vulnerability in Subversion may allow remote\n attackers to execute arbitrary code.", + "affected_packages": [ + { + "package": { + "type": "ebuild", + "namespace": "dev-vcs", + "name": "subversion", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/0.1.1|!=1.9.7", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + } + ], + "references_v2": [ + { + "reference_id": "GLSA-201709-09", + "reference_type": "", + "url": "https://security.gentoo.org/glsa/201709-09" + } + ], + "patches": [], + "severities": [ + { + "system": "generic_textual", + "value": "normal", + "scoring_elements": "" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://security.gentoo.org/glsa/201709-09" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09.xml b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09.xml new file mode 100644 index 000000000..728e6898d --- /dev/null +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09.xml @@ -0,0 +1,77 @@ + + + + Subversion: Arbitrary code execution + A command injection vulnerability in Subversion may allow remote + attackers to execute arbitrary code. + + subversion + 2017-09-17 + 2017-09-17 + 627480 + remote + + + 1.9.7 + 1.8.18 + 1.9.7 + 0.1.1 + + + + +

Subversion is a version control system intended to eventually replace + CVS. Like CVS, it has an optional client-server architecture (where the + server can be an Apache server running mod_svn, or an ssh program as in + CVS’s :ext: method). In addition to supporting the features found in + CVS, Subversion also provides support for moving and copying files and + directories. +

+ + +

Specially crafted ‘ssh://...’ URLs may allow the owner of the + repository to execute arbitrary commands on client’s machine if those + commands are already installed on the client’s system. This is + especially dangerous when the third-party repository has one or more + submodules with specially crafted ‘ssh://...’ URLs. Each time the + repository is recursively cloned or submodules are updated the payload + will be triggered. +

+ + +

A remote attacker, by enticing a user to clone a specially crafted + repository, could possibly execute arbitrary code with the privileges of + the process. +

+ + +

There are several alternative ways to fix this vulnerability. Please + refer to Subversion Team Announce for more details. +

+ + +

All Subversion 1.9.x users should upgrade to the latest version:

+ + + # emerge --sync + # emerge --ask --oneshot --verbose ">=dev-vcs/subversion-1.9.7" + + +

All Subversion 1.8.x users should upgrade to the latest version:

+ + + # emerge --sync + # emerge --ask --oneshot --verbose ">=dev-vcs/subversion-1.8.18" + + + + + CVE-2017-9800 + + + Subversion Team Announce + + + b-man + chrisadr + diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json new file mode 100644 index 000000000..5b4ac8e9e --- /dev/null +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json @@ -0,0 +1,67 @@ +[ + { + "advisory_id": "GLSA-202511-02", + "aliases": [ + "CVE-2024-40857", + "CVE-2024-40866", + "CVE-2024-44185", + "CVE-2024-44187", + "CVE-2024-44192", + "CVE-2024-44244", + "CVE-2024-44296", + "CVE-2024-54467", + "CVE-2024-54551", + "CVE-2025-24201", + "CVE-2025-24208", + "CVE-2025-24209", + "CVE-2025-24213", + "CVE-2025-24216", + "CVE-2025-24264", + "CVE-2025-30427", + "CVE-2025-31273", + "CVE-2025-31278", + "CVE-2025-43211", + "CVE-2025-43212", + "CVE-2025-43216", + "CVE-2025-43227", + "CVE-2025-43228", + "CVE-2025-43240", + "CVE-2025-43265" + ], + "summary": "Multiple vulnerabilities have been discovered in WebKitGTK+, the worst of which can lead to execution of arbitary code.", + "affected_packages": [ + { + "package": { + "type": "ebuild", + "namespace": "net-libs", + "name": "webkit-gtk", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/!=2.48.5", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + } + ], + "references_v2": [ + { + "reference_id": "GLSA-202511-02", + "reference_type": "", + "url": "https://security.gentoo.org/glsa/202511-02" + } + ], + "patches": [], + "severities": [ + { + "system": "generic_textual", + "value": "high", + "scoring_elements": "" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://security.gentoo.org/glsa/202511-02" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02.xml b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02.xml new file mode 100644 index 000000000..8f541908a --- /dev/null +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02.xml @@ -0,0 +1,73 @@ + + + + WebKitGTK+: Multiple Vulnerabilities + Multiple vulnerabilities have been discovered in WebKitGTK+, the worst of which can lead to execution of arbitary code. + webkit-gtk + 2025-11-24 + 2025-11-24 + 938026 + 941276 + 951739 + 961021 + remote + + + 2.48.5 + 2.48.5 + 2.48.5 + 2.48.5 + + + +

WebKitGTK+ is a full-featured port of the WebKit rendering engine, suitable for projects requiring any kind of web integration, from hybrid HTML/CSS applications to full-fledged web browsers.

+ + +

Multiple vulnerabilities have been discovered in WebKitGTK+. Please review the CVE identifiers referenced below for details.

+ + +

Please review the referenced CVE identifiers for details.

+ + +

There is no known workaround at this time.

+ + +

All WebKitGTK+ users should upgrade to the latest version:

+ + + # emerge --sync + # emerge --ask --oneshot --verbose ">=net-libs/webkit-gtk-2.48.5:4.1" ">=net-libs/webkit-gtk-2.48.5:6" + + + + CVE-2024-40857 + CVE-2024-40866 + CVE-2024-44185 + CVE-2024-44187 + CVE-2024-44192 + CVE-2024-44244 + CVE-2024-44296 + CVE-2024-54467 + CVE-2024-54551 + CVE-2025-24201 + CVE-2025-24208 + CVE-2025-24209 + CVE-2025-24213 + CVE-2025-24216 + CVE-2025-24264 + CVE-2025-30427 + CVE-2025-31273 + CVE-2025-31278 + CVE-2025-43211 + CVE-2025-43212 + CVE-2025-43216 + CVE-2025-43227 + CVE-2025-43228 + CVE-2025-43240 + CVE-2025-43265 + WSA-2025-0002 + WSA-2025-0003 + + graaff + sam + \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json new file mode 100644 index 000000000..38b9773c4 --- /dev/null +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json @@ -0,0 +1,41 @@ +[ + { + "advisory_id": "GLSA-202512-01", + "aliases": [], + "summary": "A vulnerability has been discovered in GnuPG, which can lead to arbitrary code execution.", + "affected_packages": [ + { + "package": { + "type": "ebuild", + "namespace": "app-crypt", + "name": "gnupg", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/!=2.5.14", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + } + ], + "references_v2": [ + { + "reference_id": "GLSA-202512-01", + "reference_type": "", + "url": "https://security.gentoo.org/glsa/202512-01" + } + ], + "patches": [], + "severities": [ + { + "system": "generic_textual", + "value": "high", + "scoring_elements": "" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://security.gentoo.org/glsa/202512-01" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01.xml b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01.xml new file mode 100644 index 000000000..d4a6b6b9d --- /dev/null +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01.xml @@ -0,0 +1,41 @@ + + + + GnuPG: Arbitrary Code Execution + A vulnerability has been discovered in GnuPG, which can lead to arbitrary code execution. + gnupg + 2025-12-27 + 2025-12-27 + 967884 + remote + + + 2.5.14 + 2.5.14 + + + +

The GNU Privacy Guard, GnuPG, is a free replacement for the PGP suite of cryptographic software.

+ + +

A vulnerability has been discovered in GnuPG's armor parser.

+ + +

A remote attacker could entice a user or automated system to process a specially crafted signature file, possibly resulting in execution of arbitrary commands with the privileges of the process.

+ + +

There is no known workaround at this time.

+ + +

All GnuPG users should upgrade to the latest version:

+ + + # emerge --sync + # emerge --ask --oneshot --verbose ">=app-crypt/gnupg-2.5.14" + + + + + sam + sam + \ No newline at end of file From 8c368b656d173724b2fdcd9f27b6c5fdb42dec81 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 12 Jan 2026 13:21:07 +0200 Subject: [PATCH 13/26] Fix Gentoo importer v1 Update the Gentoo get_safe_and_affected_versions function in advisory v2 Signed-off-by: ziad hany --- vulnerabilities/importers/gentoo.py | 24 +++-- .../pipelines/v2_importers/gentoo_importer.py | 87 +++++++++---------- .../v2_importers/test_gentoo_importer_v2.py | 9 +- .../gentoo_v2/glsa-201709-09-expected.json | 30 ++++++- .../gentoo_v2/glsa-202511-02-expected.json | 46 +++++++++- .../gentoo_v2/glsa-202512-01-expected.json | 16 +++- 6 files changed, 150 insertions(+), 62 deletions(-) diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 2f569cdf1..0f3be4431 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -6,8 +6,7 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # - - +import logging import re import xml.etree.ElementTree as ET from pathlib import Path @@ -17,12 +16,15 @@ from univers.version_constraint import VersionConstraint from univers.version_range import EbuildVersionRange from univers.versions import GentooVersion +from univers.versions import InvalidVersion from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference +logger = logging.getLogger(__name__) + class GentooImporter(Importer): repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git" @@ -104,14 +106,20 @@ def affected_and_safe_purls(affected_elem): safe_versions, affected_versions = GentooImporter.get_safe_and_affected_versions(pkg) for version in safe_versions: - constraints.append( - VersionConstraint(version=GentooVersion(version), comparator="=").invert() - ) + try: + constraints.append( + VersionConstraint(version=GentooVersion(version), comparator="=").invert() + ) + except InvalidVersion as e: + logger.error(f"Invalid safe_version {version} - error: {e}") for version in affected_versions: - constraints.append( - VersionConstraint(version=GentooVersion(version), comparator="=") - ) + try: + constraints.append( + VersionConstraint(version=GentooVersion(version), comparator="=") + ) + except InvalidVersion as e: + logger.error(f"Invalid affected_version {version} - error: {e}") if not constraints: continue diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index 61798147c..ab02a4e3f 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -121,43 +121,43 @@ def cves_from_reference(reference): return cves +def _yield_packages(pkg_name, pkg_ns, constraints, invert): + """ + Generate AffectedPackageV2 objects for a list of constraints. + """ + for comparator, version, slot_value in constraints: + qualifiers = {"slot": slot_value} if slot_value else {} + purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) + + try: + constraint = VersionConstraint(version=GentooVersion(version), comparator=comparator) + + if invert: + constraint = constraint.invert() + + yield AffectedPackageV2( + package=purl, + affected_version_range=EbuildVersionRange(constraints=[constraint]), + fixed_version_range=None, + ) + except InvalidVersion as e: + logger.error(f"InvalidVersion constraints version: {version} error:{e}") + + def affected_and_safe_purls(affected_elem): - constraints = [] for pkg in affected_elem: name = pkg.attrib.get("name") if not name: continue pkg_ns, _, pkg_name = name.rpartition("/") - purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns) - safe_versions, affected_versions = get_safe_and_affected_versions(pkg) - - for version in safe_versions: - try: - constraints.append( - VersionConstraint(version=GentooVersion(version), comparator="=").invert() - ) - except InvalidVersion as e: - logger.error(f"InvalidVersion - version: {version} - error:{e}") - - for version in affected_versions: - try: - constraints.append( - VersionConstraint(version=GentooVersion(version), comparator="=") - ) - except InvalidVersion as e: - logger.error(f"InvalidVersion - version: {version} - error:{e}") - - if not constraints: - continue - yield AffectedPackageV2( - package=purl, - affected_version_range=EbuildVersionRange(constraints=constraints), - fixed_version_range=None, - ) + safe_constraints, affected_constraints = get_safe_and_affected_constraints(pkg) + + yield from _yield_packages(pkg_name, pkg_ns, affected_constraints, invert=False) + yield from _yield_packages(pkg_name, pkg_ns, safe_constraints, invert=True) -def get_safe_and_affected_versions(pkg): +def get_safe_and_affected_constraints(pkg): # TODO : Revisit why we are skipping some versions in gentoo importer skip_versions = {"1.3*", "7.3*", "7.4*"} safe_versions = set() @@ -166,27 +166,20 @@ def get_safe_and_affected_versions(pkg): if info.text in skip_versions: continue - if info.attrib.get("range"): - if len(info.attrib.get("range")) > 2: - continue + # All possible values of info.attrib['range'] = + # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of + # which ('rle', 'rge', 'rgt') are ignored, because they compare + # 'release' not the 'version'. + range_value = info.attrib.get("range") + slot_value = info.attrib.get("slot") + comparator_dict = {"gt": ">", "lt": "<", "ge": ">=", "le": "<=", "eq": "="} + comparator = comparator_dict.get(range_value) + if not comparator: + continue if info.tag == "unaffected": - # quick hack, to know whether this - # version lies in this range, 'e' stands for - # equal, which is paired with 'greater' or 'less'. - # All possible values of info.attrib['range'] = - # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of - # which ('rle', 'rge', 'rgt') are ignored, because they compare - # 'release' not the 'version'. - if "e" in info.attrib["range"]: - safe_versions.add(info.text) - else: - affected_versions.add(info.text) + safe_versions.add((comparator, info.text, slot_value)) elif info.tag == "vulnerable": - if "e" in info.attrib["range"]: - affected_versions.add(info.text) - else: - safe_versions.add(info.text) - + affected_versions.add((comparator, info.text, slot_value)) return safe_versions, affected_versions diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py index c8206a499..796a87e1a 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py @@ -6,7 +6,7 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # - +import json from pathlib import Path from unittest.mock import Mock from unittest.mock import patch @@ -32,7 +32,10 @@ def test_gentoo_advisories_per_file(xml_file): pipeline.vcs_response = Mock(dest_dir=TEST_DATA) with patch.object(Path, "glob", return_value=[xml_file]): - result = [adv.to_dict() for adv in pipeline.collect_advisories()] + results = [adv.to_dict() for adv in pipeline.collect_advisories()] + + for adv in results: + adv["affected_packages"].sort(key=lambda x: json.dumps(x, sort_keys=True)) expected_file = xml_file.with_name(xml_file.stem + "-expected.json") - util_tests.check_results_against_json(result, expected_file) + util_tests.check_results_against_json(results, expected_file) diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json index d11fedce3..089911a0a 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -15,7 +15,35 @@ "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:ebuild/0.1.1|!=1.9.7", + "affected_version_range": "vers:ebuild/0.1.1", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "dev-vcs", + "name": "subversion", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<1.9.7", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "dev-vcs", + "name": "subversion", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<1.9.7", "fixed_version_range": null, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json index 5b4ac8e9e..f8842cc09 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json @@ -36,10 +36,52 @@ "namespace": "net-libs", "name": "webkit-gtk", "version": "", - "qualifiers": "", + "qualifiers": "slot=4.1", "subpath": "" }, - "affected_version_range": "vers:ebuild/!=2.48.5", + "affected_version_range": "vers:ebuild/<2.48.5", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "net-libs", + "name": "webkit-gtk", + "version": "", + "qualifiers": "slot=4.1", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<2.48.5", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "net-libs", + "name": "webkit-gtk", + "version": "", + "qualifiers": "slot=6", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<2.48.5", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "net-libs", + "name": "webkit-gtk", + "version": "", + "qualifiers": "slot=6", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<2.48.5", "fixed_version_range": null, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json index 38b9773c4..a4c4c3c7e 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json @@ -13,7 +13,21 @@ "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:ebuild/!=2.5.14", + "affected_version_range": "vers:ebuild/<2.5.14", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "app-crypt", + "name": "gnupg", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<2.5.14", "fixed_version_range": null, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] From 081a2451519c598f5c37d944e0144eaae3e6e5b6 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 13 Jan 2026 01:54:45 +0200 Subject: [PATCH 14/26] Fix code style Signed-off-by: ziad hany --- vulnerabilities/importers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 7c1cb85df..594021092 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -54,8 +54,8 @@ elixir_security_importer as elixir_security_importer_v2, ) from vulnerabilities.pipelines.v2_importers import epss_importer_v2 -from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2 from vulnerabilities.pipelines.v2_importers import fireeye_importer_v2 +from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2 from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2 From 782a707506b9283187fce99b4c806bce9a0e300e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 13 Jan 2026 15:18:43 +0200 Subject: [PATCH 15/26] Avoid duplicate affected_packages Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 36 +++++++++++-------- .../gentoo_v2/glsa-201709-09-expected.json | 14 -------- .../gentoo_v2/glsa-202511-02-expected.json | 28 --------------- .../gentoo_v2/glsa-202512-01-expected.json | 14 -------- 4 files changed, 22 insertions(+), 70 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index ab02a4e3f..7101ed8f2 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -64,7 +64,6 @@ def process_file(self, file): xml_root = ET.parse(file).getroot() id = xml_root.attrib.get("id") glsa = "GLSA-" + id - vuln_references = [ ReferenceV2( reference_id=glsa, @@ -82,7 +81,21 @@ def process_file(self, file): summary = child.text if child.tag == "affected": - affected_packages = list(affected_and_safe_purls(child)) + affected_packages = [] + seen_packages = set() + + for purl, constraint in get_affected_and_safe_purls(child): + signature = (purl.to_string(), str(constraint)) + + if signature not in seen_packages: + seen_packages.add(signature) + + affected_package = AffectedPackageV2( + package=purl, + affected_version_range=EbuildVersionRange(constraints=[constraint]), + fixed_version_range=None, + ) + affected_packages.append(affected_package) if child.tag == "impact": severity_value = child.attrib.get("type") @@ -121,10 +134,7 @@ def cves_from_reference(reference): return cves -def _yield_packages(pkg_name, pkg_ns, constraints, invert): - """ - Generate AffectedPackageV2 objects for a list of constraints. - """ +def extract_purls_and_constraints(pkg_name, pkg_ns, constraints, invert): for comparator, version, slot_value in constraints: qualifiers = {"slot": slot_value} if slot_value else {} purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) @@ -135,16 +145,12 @@ def _yield_packages(pkg_name, pkg_ns, constraints, invert): if invert: constraint = constraint.invert() - yield AffectedPackageV2( - package=purl, - affected_version_range=EbuildVersionRange(constraints=[constraint]), - fixed_version_range=None, - ) + yield purl, constraint except InvalidVersion as e: logger.error(f"InvalidVersion constraints version: {version} error:{e}") -def affected_and_safe_purls(affected_elem): +def get_affected_and_safe_purls(affected_elem): for pkg in affected_elem: name = pkg.attrib.get("name") if not name: @@ -153,8 +159,10 @@ def affected_and_safe_purls(affected_elem): safe_constraints, affected_constraints = get_safe_and_affected_constraints(pkg) - yield from _yield_packages(pkg_name, pkg_ns, affected_constraints, invert=False) - yield from _yield_packages(pkg_name, pkg_ns, safe_constraints, invert=True) + yield from extract_purls_and_constraints( + pkg_name, pkg_ns, affected_constraints, invert=False + ) + yield from extract_purls_and_constraints(pkg_name, pkg_ns, safe_constraints, invert=True) def get_safe_and_affected_constraints(pkg): diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json index 089911a0a..f076423cf 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -20,20 +20,6 @@ "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, - { - "package": { - "type": "ebuild", - "namespace": "dev-vcs", - "name": "subversion", - "version": "", - "qualifiers": "", - "subpath": "" - }, - "affected_version_range": "vers:ebuild/<1.9.7", - "fixed_version_range": null, - "introduced_by_commit_patches": [], - "fixed_by_commit_patches": [] - }, { "package": { "type": "ebuild", diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json index f8842cc09..2337fa9b0 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json @@ -44,34 +44,6 @@ "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, - { - "package": { - "type": "ebuild", - "namespace": "net-libs", - "name": "webkit-gtk", - "version": "", - "qualifiers": "slot=4.1", - "subpath": "" - }, - "affected_version_range": "vers:ebuild/<2.48.5", - "fixed_version_range": null, - "introduced_by_commit_patches": [], - "fixed_by_commit_patches": [] - }, - { - "package": { - "type": "ebuild", - "namespace": "net-libs", - "name": "webkit-gtk", - "version": "", - "qualifiers": "slot=6", - "subpath": "" - }, - "affected_version_range": "vers:ebuild/<2.48.5", - "fixed_version_range": null, - "introduced_by_commit_patches": [], - "fixed_by_commit_patches": [] - }, { "package": { "type": "ebuild", diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json index a4c4c3c7e..605ba19d9 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json @@ -4,20 +4,6 @@ "aliases": [], "summary": "A vulnerability has been discovered in GnuPG, which can lead to arbitrary code execution.", "affected_packages": [ - { - "package": { - "type": "ebuild", - "namespace": "app-crypt", - "name": "gnupg", - "version": "", - "qualifiers": "", - "subpath": "" - }, - "affected_version_range": "vers:ebuild/<2.5.14", - "fixed_version_range": null, - "introduced_by_commit_patches": [], - "fixed_by_commit_patches": [] - }, { "package": { "type": "ebuild", From 883947f396faca8974983073f0027f9034ce993d Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 20 Jan 2026 21:13:27 +0200 Subject: [PATCH 16/26] Avoid skipping some versions in Gentoo importer Signed-off-by: ziad hany --- vulnerabilities/pipelines/v2_importers/gentoo_importer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index 7101ed8f2..fb8b4d1d2 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -166,14 +166,9 @@ def get_affected_and_safe_purls(affected_elem): def get_safe_and_affected_constraints(pkg): - # TODO : Revisit why we are skipping some versions in gentoo importer - skip_versions = {"1.3*", "7.3*", "7.4*"} safe_versions = set() affected_versions = set() for info in pkg: - if info.text in skip_versions: - continue - # All possible values of info.attrib['range'] = # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of # which ('rle', 'rge', 'rgt') are ignored, because they compare From bf1e295a04a8a02ca6ca43725ac7e60f4e1f17c8 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 21 Jan 2026 15:06:13 +0200 Subject: [PATCH 17/26] Update gentoo importer to include rle, rgt, and rge versions. Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 18 +++++++++++------- .../gentoo_v2/glsa-201709-09-expected.json | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index fb8b4d1d2..500dc8f63 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -170,16 +170,20 @@ def get_safe_and_affected_constraints(pkg): affected_versions = set() for info in pkg: # All possible values of info.attrib['range'] = - # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of - # which ('rle', 'rge', 'rgt') are ignored, because they compare - # 'release' not the 'version'. + # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'} range_value = info.attrib.get("range") slot_value = info.attrib.get("slot") - comparator_dict = {"gt": ">", "lt": "<", "ge": ">=", "le": "<=", "eq": "="} + comparator_dict = { + "gt": ">", + "lt": "<", + "ge": ">=", + "le": "<=", + "eq": "=", + "rle": "<=", + "rge": ">=", + "rgt": ">", + } comparator = comparator_dict.get(range_value) - if not comparator: - continue - if info.tag == "unaffected": safe_versions.add((comparator, info.text, slot_value)) diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json index f076423cf..a16f92401 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -33,6 +33,20 @@ "fixed_version_range": null, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "dev-vcs", + "name": "subversion", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<=1.8.18", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] } ], "references_v2": [ From 1aa69616e3f610614f22556012e4bbec7924290a Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 2 Feb 2026 19:47:12 +0200 Subject: [PATCH 18/26] Remove invalid logger and use the pipeline logger Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index 500dc8f63..6de25206e 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -23,7 +23,6 @@ from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.management.commands.commit_export import logger from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.severity_systems import GENERIC @@ -84,7 +83,7 @@ def process_file(self, file): affected_packages = [] seen_packages = set() - for purl, constraint in get_affected_and_safe_purls(child): + for purl, constraint in get_affected_and_safe_purls(child, logger=self.log): signature = (purl.to_string(), str(constraint)) if signature not in seen_packages: @@ -134,7 +133,7 @@ def cves_from_reference(reference): return cves -def extract_purls_and_constraints(pkg_name, pkg_ns, constraints, invert): +def extract_purls_and_constraints(pkg_name, pkg_ns, constraints, invert, logger): for comparator, version, slot_value in constraints: qualifiers = {"slot": slot_value} if slot_value else {} purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) @@ -147,10 +146,10 @@ def extract_purls_and_constraints(pkg_name, pkg_ns, constraints, invert): yield purl, constraint except InvalidVersion as e: - logger.error(f"InvalidVersion constraints version: {version} error:{e}") + logger(f"InvalidVersion constraints version: {version} error:{e}") -def get_affected_and_safe_purls(affected_elem): +def get_affected_and_safe_purls(affected_elem, logger): for pkg in affected_elem: name = pkg.attrib.get("name") if not name: @@ -160,9 +159,11 @@ def get_affected_and_safe_purls(affected_elem): safe_constraints, affected_constraints = get_safe_and_affected_constraints(pkg) yield from extract_purls_and_constraints( - pkg_name, pkg_ns, affected_constraints, invert=False + pkg_name, pkg_ns, affected_constraints, invert=False, logger=logger + ) + yield from extract_purls_and_constraints( + pkg_name, pkg_ns, safe_constraints, invert=True, logger=logger ) - yield from extract_purls_and_constraints(pkg_name, pkg_ns, safe_constraints, invert=True) def get_safe_and_affected_constraints(pkg): From f1efbbad062a49a65e22eb8d04390f4effc07339 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 6 Feb 2026 03:23:01 +0200 Subject: [PATCH 19/26] Fix minor issue related to advisory url and safe default for id Signed-off-by: ziad hany --- vulnerabilities/pipelines/v2_importers/gentoo_importer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index 6de25206e..806200572 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -61,7 +61,7 @@ def process_file(self, file): cves = [] summary = "" xml_root = ET.parse(file).getroot() - id = xml_root.attrib.get("id") + id = xml_root.attrib.get("id", "") glsa = "GLSA-" + id vuln_references = [ ReferenceV2( @@ -108,9 +108,7 @@ def process_file(self, file): references_v2=vuln_references, severities=severities, affected_packages=affected_packages, - url=f"https://security.gentoo.org/glsa/{id}" - if id - else "https://security.gentoo.org/glsa", + url=f"https://security.gentoo.org/glsa/{id}", original_advisory_text=file, ) From 07b25e999b335bc375bdc2e5c2b98f0f0b849d76 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 9 Feb 2026 13:59:47 +0200 Subject: [PATCH 20/26] Update the Gentoo importer to correctly handle affected_version_range and fixed_version_range for non-revision versions Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 132 ++++++++++-------- .../gentoo_v2/glsa-201709-09-expected.json | 32 +---- .../gentoo_v2/glsa-202511-02-expected.json | 4 +- .../gentoo_v2/glsa-202512-01-expected.json | 2 +- 4 files changed, 76 insertions(+), 94 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index 806200572..e61732ecc 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -9,6 +9,7 @@ import re import xml.etree.ElementTree as ET +from collections import defaultdict from pathlib import Path from typing import Iterable @@ -81,20 +82,23 @@ def process_file(self, file): if child.tag == "affected": affected_packages = [] - seen_packages = set() - - for purl, constraint in get_affected_and_safe_purls(child, logger=self.log): - signature = (purl.to_string(), str(constraint)) - - if signature not in seen_packages: - seen_packages.add(signature) - - affected_package = AffectedPackageV2( - package=purl, - affected_version_range=EbuildVersionRange(constraints=[constraint]), - fixed_version_range=None, - ) - affected_packages.append(affected_package) + for purl, (affected_ranges, fixed_ranges) in get_affected_and_fixed_purls( + child, logger=self.log + ): + affected_version_constraint = build_constraints( + affected_ranges, logger=self.log + ) + fixed_version_constraint = build_constraints(fixed_ranges, logger=self.log) + affected_version_range = EbuildVersionRange( + constraints=affected_version_constraint + ) + fixed_version_range = EbuildVersionRange(constraints=fixed_version_constraint) + affected_package = AffectedPackageV2( + package=purl, + affected_version_range=affected_version_range, + fixed_version_range=fixed_version_range, + ) + affected_packages.append(affected_package) if child.tag == "impact": severity_value = child.attrib.get("type") @@ -131,61 +135,67 @@ def cves_from_reference(reference): return cves -def extract_purls_and_constraints(pkg_name, pkg_ns, constraints, invert, logger): - for comparator, version, slot_value in constraints: - qualifiers = {"slot": slot_value} if slot_value else {} - purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) - +def build_constraints(constraint_pairs, logger): + """ + Build a list of VersionConstraint objects from comparators, versions pairs. + """ + constraints = [] + for comparator, version in constraint_pairs: try: constraint = VersionConstraint(version=GentooVersion(version), comparator=comparator) - - if invert: - constraint = constraint.invert() - - yield purl, constraint + constraints.append(constraint) except InvalidVersion as e: logger(f"InvalidVersion constraints version: {version} error:{e}") + return constraints + +def get_affected_and_fixed_purls(affected_elem, logger): + """ + Parses XML elements to extract PURLs associated with affected and fixed versions. + """ -def get_affected_and_safe_purls(affected_elem, logger): for pkg in affected_elem: name = pkg.attrib.get("name") if not name: continue - pkg_ns, _, pkg_name = name.rpartition("/") - - safe_constraints, affected_constraints = get_safe_and_affected_constraints(pkg) - yield from extract_purls_and_constraints( - pkg_name, pkg_ns, affected_constraints, invert=False, logger=logger - ) - yield from extract_purls_and_constraints( - pkg_name, pkg_ns, safe_constraints, invert=True, logger=logger - ) - - -def get_safe_and_affected_constraints(pkg): - safe_versions = set() - affected_versions = set() - for info in pkg: - # All possible values of info.attrib['range'] = - # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'} - range_value = info.attrib.get("range") - slot_value = info.attrib.get("slot") - comparator_dict = { - "gt": ">", - "lt": "<", - "ge": ">=", - "le": "<=", - "eq": "=", - "rle": "<=", - "rge": ">=", - "rgt": ">", - } - comparator = comparator_dict.get(range_value) - if info.tag == "unaffected": - safe_versions.add((comparator, info.text, slot_value)) - - elif info.tag == "vulnerable": - affected_versions.add((comparator, info.text, slot_value)) - return safe_versions, affected_versions + pkg_ns, _, pkg_name = name.rpartition("/") + # purl_components, (fixed_ranges_set, affected_ranges_set) + purl_ranges_map = defaultdict(lambda: {"fixed_ranges": set(), "affected_ranges": set()}) + + for info in pkg: + # All possible values of info.attrib['range'] = + # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'} + # rge means revision greater than equals and rgt means revision greater than + + range_value = info.attrib.get("range") + slot_value = info.attrib.get("slot") + comparator_dict = { + "gt": ">", + "lt": "<", + "ge": ">=", + "le": "<=", + "eq": "=", + # "rle": "<=", + # "rge": ">=", + # "rgt": ">", + } + comparator = comparator_dict.get(range_value) + if not comparator: + logger(f"Unsupported range value {range_value}:{info.text}") + continue + + if info.tag == "unaffected": + purl_ranges_map[(pkg_name, pkg_ns, slot_value)]["fixed_ranges"].add( + (comparator, info.text) + ) + + elif info.tag == "vulnerable": + purl_ranges_map[(pkg_name, pkg_ns, slot_value)]["affected_ranges"].add( + (comparator, info.text) + ) + + for (pkg_name, pkg_ns, slot_value), data in purl_ranges_map.items(): + qualifiers = {"slot": slot_value} if slot_value else {} + purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) + yield purl, (data["affected_ranges"], data["fixed_ranges"]) diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json index a16f92401..f05f035a2 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -15,36 +15,8 @@ "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:ebuild/0.1.1", - "fixed_version_range": null, - "introduced_by_commit_patches": [], - "fixed_by_commit_patches": [] - }, - { - "package": { - "type": "ebuild", - "namespace": "dev-vcs", - "name": "subversion", - "version": "", - "qualifiers": "", - "subpath": "" - }, - "affected_version_range": "vers:ebuild/<1.9.7", - "fixed_version_range": null, - "introduced_by_commit_patches": [], - "fixed_by_commit_patches": [] - }, - { - "package": { - "type": "ebuild", - "namespace": "dev-vcs", - "name": "subversion", - "version": "", - "qualifiers": "", - "subpath": "" - }, - "affected_version_range": "vers:ebuild/<=1.8.18", - "fixed_version_range": null, + "affected_version_range": "vers:ebuild/0.1.1|<1.9.7", + "fixed_version_range": "vers:ebuild/>=1.9.7", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json index 2337fa9b0..e1d909246 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json @@ -40,7 +40,7 @@ "subpath": "" }, "affected_version_range": "vers:ebuild/<2.48.5", - "fixed_version_range": null, + "fixed_version_range": "vers:ebuild/>=2.48.5", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -54,7 +54,7 @@ "subpath": "" }, "affected_version_range": "vers:ebuild/<2.48.5", - "fixed_version_range": null, + "fixed_version_range": "vers:ebuild/>=2.48.5", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json index 605ba19d9..00c972dc0 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json @@ -14,7 +14,7 @@ "subpath": "" }, "affected_version_range": "vers:ebuild/<2.5.14", - "fixed_version_range": null, + "fixed_version_range": "vers:ebuild/>=2.5.14", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } From 478f8c5c2a71c82497be40eca27fac98ea2295be Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 11 Feb 2026 16:46:21 +0200 Subject: [PATCH 21/26] Update gentoo to support rgt,rge,rle version ranges Update the pipeline to use the new AdvisoryDataV2 Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 21 ++++++++++++------- .../gentoo_v2/glsa-201709-09-expected.json | 4 ++-- .../gentoo_v2/glsa-202511-02-expected.json | 2 +- .../gentoo_v2/glsa-202512-01-expected.json | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index e61732ecc..6d1bc3d3f 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -20,7 +20,7 @@ from univers.versions import GentooVersion from univers.versions import InvalidVersion -from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity @@ -53,7 +53,7 @@ def advisories_count(self): advisory_dir = Path(self.vcs_response.dest_dir) return sum(1 for _ in advisory_dir.rglob("*.xml")) - def collect_advisories(self) -> Iterable[AdvisoryData]: + def collect_advisories(self) -> Iterable[AdvisoryDataV2]: base_path = Path(self.vcs_response.dest_dir) for file_path in base_path.glob("**/*.xml"): yield from self.process_file(file_path) @@ -105,11 +105,11 @@ def process_file(self, file): if severity_value: severities.append(VulnerabilitySeverity(system=GENERIC, value=severity_value)) - yield AdvisoryData( + yield AdvisoryDataV2( advisory_id=glsa, aliases=cves, summary=summary, - references_v2=vuln_references, + references=vuln_references, severities=severities, affected_packages=affected_packages, url=f"https://security.gentoo.org/glsa/{id}", @@ -176,9 +176,9 @@ def get_affected_and_fixed_purls(affected_elem, logger): "ge": ">=", "le": "<=", "eq": "=", - # "rle": "<=", - # "rge": ">=", - # "rgt": ">", + "rle": "<=", + "rge": ">=", + "rgt": ">", } comparator = comparator_dict.get(range_value) if not comparator: @@ -195,6 +195,13 @@ def get_affected_and_fixed_purls(affected_elem, logger): (comparator, info.text) ) + if range_value in ["rgt", "rge", "rle"]: + next_minor_version = GentooVersion(info.text).bump() + invert_comp = "<" if range_value in ["rgt", "rge"] else ">" + purl_ranges_map[(pkg_name, pkg_ns, slot_value)]["fixed_ranges"].add( + (invert_comp, next_minor_version) + ) + for (pkg_name, pkg_ns, slot_value), data in purl_ranges_map.items(): qualifiers = {"slot": slot_value} if slot_value else {} purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json index f05f035a2..9e6925d35 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -16,12 +16,12 @@ "subpath": "" }, "affected_version_range": "vers:ebuild/0.1.1|<1.9.7", - "fixed_version_range": "vers:ebuild/>=1.9.7", + "fixed_version_range": "vers:ebuild/>1.8.18|<1.9|>=1.9.7", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } ], - "references_v2": [ + "references": [ { "reference_id": "GLSA-201709-09", "reference_type": "", diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json index e1d909246..18617a5ad 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json @@ -59,7 +59,7 @@ "fixed_by_commit_patches": [] } ], - "references_v2": [ + "references": [ { "reference_id": "GLSA-202511-02", "reference_type": "", diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json index 00c972dc0..1d889dd80 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json @@ -19,7 +19,7 @@ "fixed_by_commit_patches": [] } ], - "references_v2": [ + "references": [ { "reference_id": "GLSA-202512-01", "reference_type": "", From d30d9f3bd93aea306bba5bb4d32c5a6a31d7e861 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 18 Feb 2026 13:21:54 +0200 Subject: [PATCH 22/26] Update gentoo to create an affected_package for every index range Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 63 ++++++++----------- .../gentoo_v2/glsa-201709-09-expected.json | 46 +++++++++++++- .../gentoo_v2/glsa-202511-02-expected.json | 30 ++++++++- .../gentoo_v2/glsa-202512-01-expected.json | 14 +++++ 4 files changed, 112 insertions(+), 41 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index 6d1bc3d3f..32970f092 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -9,7 +9,6 @@ import re import xml.etree.ElementTree as ET -from collections import defaultdict from pathlib import Path from typing import Iterable @@ -82,22 +81,23 @@ def process_file(self, file): if child.tag == "affected": affected_packages = [] - for purl, (affected_ranges, fixed_ranges) in get_affected_and_fixed_purls( + for purl, constraints, is_unaffected in get_affected_and_fixed_purls( child, logger=self.log ): - affected_version_constraint = build_constraints( - affected_ranges, logger=self.log - ) - fixed_version_constraint = build_constraints(fixed_ranges, logger=self.log) - affected_version_range = EbuildVersionRange( - constraints=affected_version_constraint - ) - fixed_version_range = EbuildVersionRange(constraints=fixed_version_constraint) - affected_package = AffectedPackageV2( - package=purl, - affected_version_range=affected_version_range, - fixed_version_range=fixed_version_range, - ) + constraints = build_constraints(constraints, logger=self.log) + version_range = EbuildVersionRange(constraints=constraints) + + if is_unaffected: + affected_package = AffectedPackageV2( + package=purl, + fixed_version_range=version_range, + ) + else: + affected_package = AffectedPackageV2( + package=purl, + affected_version_range=version_range, + ) + affected_packages.append(affected_package) if child.tag == "impact": @@ -160,14 +160,10 @@ def get_affected_and_fixed_purls(affected_elem, logger): continue pkg_ns, _, pkg_name = name.rpartition("/") - # purl_components, (fixed_ranges_set, affected_ranges_set) - purl_ranges_map = defaultdict(lambda: {"fixed_ranges": set(), "affected_ranges": set()}) - for info in pkg: # All possible values of info.attrib['range'] = # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'} # rge means revision greater than equals and rgt means revision greater than - range_value = info.attrib.get("range") slot_value = info.attrib.get("slot") comparator_dict = { @@ -185,24 +181,15 @@ def get_affected_and_fixed_purls(affected_elem, logger): logger(f"Unsupported range value {range_value}:{info.text}") continue - if info.tag == "unaffected": - purl_ranges_map[(pkg_name, pkg_ns, slot_value)]["fixed_ranges"].add( - (comparator, info.text) - ) - - elif info.tag == "vulnerable": - purl_ranges_map[(pkg_name, pkg_ns, slot_value)]["affected_ranges"].add( - (comparator, info.text) - ) - - if range_value in ["rgt", "rge", "rle"]: - next_minor_version = GentooVersion(info.text).bump() - invert_comp = "<" if range_value in ["rgt", "rge"] else ">" - purl_ranges_map[(pkg_name, pkg_ns, slot_value)]["fixed_ranges"].add( - (invert_comp, next_minor_version) - ) - - for (pkg_name, pkg_ns, slot_value), data in purl_ranges_map.items(): qualifiers = {"slot": slot_value} if slot_value else {} purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) - yield purl, (data["affected_ranges"], data["fixed_ranges"]) + + constraints = [(comparator, info.text)] + if range_value in ["rgt", "rge", "rle"]: + try: + next_minor_version = str(GentooVersion(info.text).bump()) + invert_comp = "<" if range_value in ["rgt", "rge"] else ">" + constraints.append((invert_comp, next_minor_version)) + except Exception as e: + logger(f"Invalid Gentoo version for bumping: {info.text} - {e}") + yield purl, constraints, (info.tag == "unaffected") diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json index 9e6925d35..e4f3a0551 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -15,8 +15,50 @@ "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:ebuild/0.1.1|<1.9.7", - "fixed_version_range": "vers:ebuild/>1.8.18|<1.9|>=1.9.7", + "affected_version_range": "vers:ebuild/0.1.1", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "dev-vcs", + "name": "subversion", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:ebuild/<1.9.7", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "dev-vcs", + "name": "subversion", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": null, + "fixed_version_range": "vers:ebuild/>1.8.18|<1.9", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "dev-vcs", + "name": "subversion", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": null, + "fixed_version_range": "vers:ebuild/>=1.9.7", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json index 18617a5ad..b5d4e849b 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202511-02-expected.json @@ -40,7 +40,7 @@ "subpath": "" }, "affected_version_range": "vers:ebuild/<2.48.5", - "fixed_version_range": "vers:ebuild/>=2.48.5", + "fixed_version_range": null, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -54,6 +54,34 @@ "subpath": "" }, "affected_version_range": "vers:ebuild/<2.48.5", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "net-libs", + "name": "webkit-gtk", + "version": "", + "qualifiers": "slot=4.1", + "subpath": "" + }, + "affected_version_range": null, + "fixed_version_range": "vers:ebuild/>=2.48.5", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "net-libs", + "name": "webkit-gtk", + "version": "", + "qualifiers": "slot=6", + "subpath": "" + }, + "affected_version_range": null, "fixed_version_range": "vers:ebuild/>=2.48.5", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json index 1d889dd80..c15394806 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-202512-01-expected.json @@ -14,6 +14,20 @@ "subpath": "" }, "affected_version_range": "vers:ebuild/<2.5.14", + "fixed_version_range": null, + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "ebuild", + "namespace": "app-crypt", + "name": "gnupg", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": null, "fixed_version_range": "vers:ebuild/>=2.5.14", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] From baa44ae19bd4f388f9283d84b401b97f9adbfe65 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 19 Feb 2026 01:24:48 +0200 Subject: [PATCH 23/26] Update Gentoo importer to treat revision versions as a regular version. Signed-off-by: ziad hany --- .../pipelines/v2_importers/gentoo_importer.py | 14 +++----------- .../gentoo_v2/glsa-201709-09-expected.json | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index 32970f092..dbed04c90 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -84,7 +84,7 @@ def process_file(self, file): for purl, constraints, is_unaffected in get_affected_and_fixed_purls( child, logger=self.log ): - constraints = build_constraints(constraints, logger=self.log) + constraints = build_constraints([constraints], logger=self.log) version_range = EbuildVersionRange(constraints=constraints) if is_unaffected: @@ -164,6 +164,7 @@ def get_affected_and_fixed_purls(affected_elem, logger): # All possible values of info.attrib['range'] = # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'} # rge means revision greater than equals and rgt means revision greater than + # TODO Revisit issue: range_value = info.attrib.get("range") slot_value = info.attrib.get("slot") comparator_dict = { @@ -183,13 +184,4 @@ def get_affected_and_fixed_purls(affected_elem, logger): qualifiers = {"slot": slot_value} if slot_value else {} purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) - - constraints = [(comparator, info.text)] - if range_value in ["rgt", "rge", "rle"]: - try: - next_minor_version = str(GentooVersion(info.text).bump()) - invert_comp = "<" if range_value in ["rgt", "rge"] else ">" - constraints.append((invert_comp, next_minor_version)) - except Exception as e: - logger(f"Invalid Gentoo version for bumping: {info.text} - {e}") - yield purl, constraints, (info.tag == "unaffected") + yield purl, (comparator, info.text), (info.tag == "unaffected") diff --git a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json index e4f3a0551..684d49bc5 100644 --- a/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json +++ b/vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json @@ -44,7 +44,7 @@ "subpath": "" }, "affected_version_range": null, - "fixed_version_range": "vers:ebuild/>1.8.18|<1.9", + "fixed_version_range": "vers:ebuild/>1.8.18", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, From 0ab34197023ba9278c09e08e0fdcc940c491fa87 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 19 Feb 2026 02:12:52 +0200 Subject: [PATCH 24/26] Fix a typo in the gentoo importer docs Signed-off-by: ziad hany --- vulnerabilities/pipelines/v2_importers/gentoo_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py index dbed04c90..5db00a649 100644 --- a/vulnerabilities/pipelines/v2_importers/gentoo_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gentoo_importer.py @@ -164,7 +164,7 @@ def get_affected_and_fixed_purls(affected_elem, logger): # All possible values of info.attrib['range'] = # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'} # rge means revision greater than equals and rgt means revision greater than - # TODO Revisit issue: + # TODO Revisit issue: https://github.com/aboutcode-org/vulnerablecode/issues/2180 range_value = info.attrib.get("range") slot_value = info.attrib.get("slot") comparator_dict = { From 5406456df605c0ba0dfafd722c19d8d017dcf147 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 19 Feb 2026 16:38:40 +0200 Subject: [PATCH 25/26] Resolve SyntaxWarning in Fireeye Importer Signed-off-by: ziad hany --- vulnerabilities/importers/fireeye.py | 2 +- vulnerabilities/pipelines/v2_importers/fireeye_importer_v2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py index 03fb3a8d5..404f2d367 100644 --- a/vulnerabilities/importers/fireeye.py +++ b/vulnerabilities/importers/fireeye.py @@ -112,7 +112,7 @@ def matcher_url(ref) -> str: """ Returns URL of the reference markup from reference url in Markdown format """ - markup_regex = "\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)" + markup_regex = r"\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)" matched_markup = re.findall(markup_regex, ref) if matched_markup: return matched_markup[0][1] diff --git a/vulnerabilities/pipelines/v2_importers/fireeye_importer_v2.py b/vulnerabilities/pipelines/v2_importers/fireeye_importer_v2.py index 89c3875ed..b1283d132 100644 --- a/vulnerabilities/pipelines/v2_importers/fireeye_importer_v2.py +++ b/vulnerabilities/pipelines/v2_importers/fireeye_importer_v2.py @@ -154,7 +154,7 @@ def matcher_url(ref) -> str: """ Returns URL of the reference markup from reference url in Markdown format """ - markup_regex = "\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)" + markup_regex = r"\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)" matched_markup = re.findall(markup_regex, ref) if matched_markup: return matched_markup[0][1] From c753d276cbd13c4c98968ee4bb13e40c9f593763 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 20 Feb 2026 16:39:28 +0530 Subject: [PATCH 26/26] Use related advisory severity to calculate exploitibility, weighted severity and risk scores Signed-off-by: Tushar Goel --- .../v2_improvers/compute_package_risk.py | 18 ++++++++++++++++-- .../v2_improvers/test_relate_severities.py | 6 ++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index ac7caa49d..9caaaeb95 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -7,7 +7,10 @@ # See https://aboutcode.org for more information about nexB OSS projects. # from aboutcode.pipeline import LoopProgress +from django.db.models import Prefetch +from django.db.models import Q +from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline @@ -35,7 +38,15 @@ def steps(cls): def compute_and_store_vulnerability_risk_score(self): affected_advisories = ( AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False) - .prefetch_related("references", "severities", "exploits") + .prefetch_related( + "references", + "severities", + "exploits", + Prefetch( + "related_advisory_severities", + queryset=AdvisoryV2.objects.prefetch_related("severities"), + ), + ) .distinct() ) @@ -50,10 +61,13 @@ def compute_and_store_vulnerability_risk_score(self): batch_size = 5000 for advisory in progress.iter(affected_advisories.iterator(chunk_size=batch_size)): - severities = advisory.severities.all() references = advisory.references.all() exploits = advisory.exploits.all() + severities = AdvisorySeverity.objects.filter( + Q(advisories=advisory) | Q(advisories__related_to_advisory_severities=advisory) + ).distinct() + weighted_severity, exploitability = compute_vulnerability_risk_factors( references=references, severities=severities, diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_relate_severities.py b/vulnerabilities/tests/pipelines/v2_improvers/test_relate_severities.py index 0c4c3e901..2dadbc679 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_relate_severities.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_relate_severities.py @@ -8,7 +8,9 @@ # import pytest +from django.db.models import Q +from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.pipelines.v2_improvers.relate_severities import RelateSeveritiesPipeline from vulnerabilities.severity_systems import EPSS @@ -42,6 +44,10 @@ def test_relate_severities_by_advisory_id(): pipeline.relate_severities() assert base.related_advisory_severities.filter(id=severity_advisory.id).exists() + severities = AdvisorySeverity.objects.filter( + Q(advisories=base) | Q(advisories__related_to_advisory_severities=base) + ).distinct() + assert severities.filter(id=severity_advisory.severities.first().id).exists() @pytest.mark.django_db